В справке написано:
Qt::PartiallyChecked: The item is partially checked. Items in hierarchical models may be partially checked if some, but not all, of their children are checked.
Дабовь к этому элементу дочерних итемов с чекабельностью, например так:     
    QTreeWidgetItem *wi = new QTreeWidgetItem(treeWidget);
    treeWidget->setEditTriggers(QAbstractItemView::AllEditTriggers);
    wi->setText(0, "test");
    wi->setFlags(Qt::ItemIsEnabled | Qt::ItemIsTristate | Qt::ItemIsUserCheckable);
    wi->setCheckState(0, Qt::PartiallyChecked);
    QTreeWidgetItem *wi2 = new QTreeWidgetItem(wi);
    wi2->setText(0, "wi2");
    QTreeWidgetItem *wi3 = new QTreeWidgetItem(wi);
    wi3->setText(0, "wi3");
    wi2->setFlags(Qt::ItemIsEnabled | Qt::ItemIsTristate | Qt::ItemIsUserCheckable);
    wi3->setFlags(Qt::ItemIsEnabled | Qt::ItemIsTristate | Qt::ItemIsUserCheckable);
    wi2->setCheckState(0, Qt::Unchecked);
    wi3->setCheckState(0, Qt::Unchecked);