Skip to content

Commit 4673160

Browse files
gesinn-it-evlgesinn-it-evl
authored
Merge pull request #11 from gesinn-it-pub/UpdateJob
Update job
2 parents 14a6a64 + f5436c3 commit 4673160

File tree

4 files changed

+148
-212
lines changed

4 files changed

+148
-212
lines changed

Diff for: ‎extension.json

-4
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,6 @@
2525
"SDU\\Hooks::onAfterDataUpdateComplete"
2626
]
2727
},
28-
"JobClasses": {
29-
"RebuildDataJob": "SDU\\RebuildDataJob",
30-
"PageUpdaterJob": "SDU\\PageUpdaterJob"
31-
},
3228
"callback": "SDU\\Hooks::setup",
3329
"config": {
3430
"SDUProperty": "Semantic Dependency",

Diff for: ‎src/Hooks.php

+148-136
Original file line numberDiff line numberDiff line change
@@ -4,180 +4,192 @@
44

55
use DeferredUpdates;
66
use JobQueueGroup;
7-
use SMW\Options;
7+
use SMW\MediaWiki\Jobs\UpdateJob;
88
use SMW\Services\ServicesFactory as ApplicationFactory;
99
use SMWDIBlob;
1010
use SMWQueryProcessor;
1111
use SMWSemanticData;
1212
use SMWStore;
13-
use WikiPage;
1413

1514
class Hooks {
1615

17-
public static function setup() {
18-
if ( !defined( 'MEDIAWIKI' ) ) {
19-
die();
20-
}
16+
public static function setup() {
17+
if ( !defined( 'MEDIAWIKI' ) ) {
18+
die();
19+
}
2120

22-
if ( !defined( 'SMW_VERSION' ) ) {
23-
die( "ERROR: Semantic MediaWiki must be installed for Semantic Dependency Updater to run!" );
24-
}
21+
if ( !defined( 'SMW_VERSION' ) ) {
22+
die( "ERROR: Semantic MediaWiki must be installed for Semantic Dependency Updater to run!" );
2523
}
24+
}
2625

27-
public static function onAfterDataUpdateComplete(
28-
SMWStore $store, SMWSemanticData $newData,
29-
$compositePropertyTableDiffIterator
30-
) {
31-
global $wgSDUProperty;
32-
global $wgSDUTraversed;
26+
public static function onAfterDataUpdateComplete(
27+
SMWStore $store, SMWSemanticData $newData,
28+
$compositePropertyTableDiffIterator
29+
) {
30+
global $wgSDUProperty;
31+
global $wgSDUTraversed;
3332

34-
if ( !isset( $wgSDUTraversed ) ) {
35-
$wgSDUTraversed = [];
36-
}
33+
if ( !isset( $wgSDUTraversed ) ) {
34+
$wgSDUTraversed = [];
35+
}
3736

38-
$wgSDUProperty = str_replace( ' ', '_', $wgSDUProperty );
39-
$subject = $newData->getSubject();
40-
$title = $subject->getTitle();
41-
if ( $title == null ) {
42-
return true;
43-
}
37+
$wgSDUProperty = str_replace( ' ', '_', $wgSDUProperty );
38+
$subject = $newData->getSubject();
39+
$title = $subject->getTitle();
40+
if ( $title == null ) {
41+
return true;
42+
}
4443

45-
$id = $title->getPrefixedDBKey();
44+
$id = $title->getPrefixedDBKey();
4645

47-
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --> " . $title );
46+
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --> " . $title );
4847

49-
// FIRST CHECK: Does the page data contain a $wgSDUProperty semantic property ?
50-
$properties = $newData->getProperties();
51-
if ( !isset( $properties[$wgSDUProperty] ) ) {
52-
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- No SDU property found" );
53-
return true;
54-
}
48+
// FIRST CHECK: Does the page data contain a $wgSDUProperty semantic property ?
49+
$properties = $newData->getProperties();
50+
if ( !isset( $properties[$wgSDUProperty] ) ) {
51+
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- No SDU property found" );
52+
return true;
53+
}
5554

56-
$diffTable = $compositePropertyTableDiffIterator->getOrderedDiffByTable();
55+
$diffTable = $compositePropertyTableDiffIterator->getOrderedDiffByTable();
56+
$smwSID = $compositePropertyTableDiffIterator->getSubject()->getId();
57+
// SECOND CHECK: Have there been actual changes in the data? (Ignore internal SMW data!)
58+
// TODO: Introduce an explicit list of Semantic Properties to watch ?
59+
unset( $diffTable['smw_fpt_mdat'] ); // Ignore SMW's internal properties "smw_fpt_mdat"
60+
61+
// lets try first to check the data tables: https://www.semantic-mediawiki.org/wiki/Help:Database_schema
62+
// if change, on pageID from Issue, that is not REvision ID, then trigger all changes
63+
$triggerSemanticDependencies = false;
64+
65+
if ( count( $diffTable ) > 0 ) {
66+
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] -----> Data changes detected" );
67+
68+
foreach ( $diffTable as $key => $value ) {
69+
if ( strpos( $key, 'smw_di' ) === 0 && is_array( $value ) ) {
70+
foreach ( $value["insert"] as $insert ) {
71+
if ( $insert["s_id"] == $smwSID ) {
72+
if ( $insert["p_id"] != 506 ) {
73+
$triggerSemanticDependencies = true;
74+
break 2;
75+
} // revision ID change is good, but must not trigger UpdateJob for semantic dependencies
76+
}
77+
}
78+
echo $key;
79+
}
80+
}
81+
} else {
82+
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- No semantic data changes detected" );
83+
return true;
84+
}
5785

58-
// SECOND CHECK: Have there been actual changes in the data? (Ignore internal SMW data!)
59-
// TODO: Introduce an explicit list of Semantic Properties to watch ?
60-
unset( $diffTable['smw_fpt_mdat'] ); // Ignore SMW's internal properties "smw_fpt_mdat"
86+
// THIRD CHECK: Has this page been already traversed more than twice?
87+
// This should only be the case when SMW errors occur.
88+
// In that case, the diffTable contains everything and SDU can't know if changes happened
89+
if ( array_key_exists( $id, $wgSDUTraversed ) ) {
90+
$wgSDUTraversed[$id] = $wgSDUTraversed[$id] + 1;
91+
} else {
92+
$wgSDUTraversed[$id] = 1;
93+
}
94+
if ( $wgSDUTraversed[$id] > 2 ) {
95+
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- Already traversed" );
96+
return true;
97+
}
6198

62-
if ( count( $diffTable ) > 0 ) {
63-
// wfDebugLog('SemanticDependencyUpdater', "[SDU] diffTable: " . print_r($diffTable, true));
64-
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] -----> Data changes detected" );
65-
} else {
66-
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- No semantic data changes detected" );
67-
return true;
99+
// QUERY AND UPDATE DEPENDENCIES
100+
101+
// SMW\SemanticData $newData
102+
// SMWDataItem[] $dataItem
103+
$wikiPageValues = [];
104+
if ( $triggerSemanticDependencies ) {
105+
$dataItem = $newData->getPropertyValues( $properties[$wgSDUProperty] );
106+
if ( $dataItem != null ) {
107+
foreach ( $dataItem as $valueItem ) {
108+
if ( $valueItem instanceof SMWDIBlob && $valueItem->getString() != $id ) {
109+
$wikiPageValues = array_merge( $wikiPageValues, self::updatePagesMatchingQuery( $valueItem->getSerialization() ) );
110+
}
68111
}
112+
}
113+
} else {
114+
$wikiPageValues = [ $subject ];
115+
}
69116

70-
// THIRD CHECK: Has this page been already traversed more than twice?
71-
// This should only be the case when SMW errors occur.
72-
// In that case, the diffTable contains everything and SDU can't know if changes happened
73-
if ( array_key_exists( $id, $wgSDUTraversed ) ) {
74-
$wgSDUTraversed[$id] = $wgSDUTraversed[$id] + 1;
75-
} else {
76-
$wgSDUTraversed[$id] = 1;
77-
}
78-
if ( $wgSDUTraversed[$id] > 2 ) {
79-
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] <-- Already traversed" );
80-
return true;
81-
}
117+
self::rebuildData( $triggerSemanticDependencies, $wikiPageValues, $subject );
82118

83-
// QUERY AND UPDATE DEPENDENCIES
119+
return true;
120+
}
84121

85-
// SMW\SemanticData $newData
86-
// SMWDataItem[] $dataItem
87-
$dataItem = $newData->getPropertyValues( $properties[$wgSDUProperty] );
122+
/**
123+
* @param string $queryString Query string, excluding [[ and ]] brackets
124+
*/
125+
private static function updatePagesMatchingQuery( $queryString ) {
126+
global $sfgListSeparator;
88127

89-
$wikiPageValues = [];
90-
if ( $dataItem != null ) {
91-
foreach ( $dataItem as $valueItem ) {
92-
if ( $valueItem instanceof SMWDIBlob ) {
93-
$wikiPageValues = array_merge( $wikiPageValues, self::updatePagesMatchingQuery( $valueItem->getSerialization() ) );
94-
}
95-
}
96-
}
128+
$queryString = str_replace( 'AND', ']] [[', $queryString );
129+
$queryString = str_replace( 'OR', ']] OR [[', $queryString );
97130

98-
self::rebuildData( $wikiPageValues, $store );
99-
return true;
131+
// If SF is installed, get the separator character and change it into ||
132+
// Otherwise SDU won't work with multi-value properties
133+
if ( isset( $sfgListSeparator ) ) {
134+
$queryString = rtrim( $queryString, $sfgListSeparator );
135+
$queryString = str_replace( $sfgListSeparator, ' || ', $queryString );
100136
}
101137

102-
/**
103-
* @param string $queryString Query string, excluding [[ and ]] brackets
104-
*/
105-
private static function updatePagesMatchingQuery( $queryString ) {
106-
global $sfgListSeparator;
138+
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --------> [[$queryString]]" );
107139

108-
$queryString = str_replace( 'AND', ']] [[', $queryString );
109-
$queryString = str_replace( 'OR', ']] OR [[', $queryString );
140+
$store = smwfGetStore();
110141

111-
// If SF is installed, get the separator character and change it into ||
112-
// Otherwise SDU won't work with multi-value properties
113-
if ( isset( $sfgListSeparator ) ) {
114-
$queryString = rtrim( $queryString, $sfgListSeparator );
115-
$queryString = str_replace( $sfgListSeparator, ' || ', $queryString );
116-
}
142+
$params = [
143+
'limit' => 10000,
144+
];
145+
$processedParams = SMWQueryProcessor::getProcessedParams( $params );
146+
$query =
147+
SMWQueryProcessor::createQuery( "[[$queryString]]", $processedParams, SMWQueryProcessor::SPECIAL_PAGE );
148+
$result = $store->getQueryResult( $query );
149+
$wikiPageValues = $result->getResults();
117150

118-
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --------> [[$queryString]]" );
151+
return $wikiPageValues;
152+
}
119153

120-
$store = smwfGetStore();
154+
/**
155+
* Rebuilds data of the given wikipages to regenerate semantic attrubutes and re-run queries
156+
*/
157+
public static function rebuildData( $triggerSemanticDependencies, $wikiPageValues, $subject ) {
158+
global $wgSDUUseJobQueue;
121159

122-
$params = [
123-
'limit' => 10000,
124-
];
125-
$processedParams = SMWQueryProcessor::getProcessedParams( $params );
126-
$query =
127-
SMWQueryProcessor::createQuery( "[[$queryString]]", $processedParams, SMWQueryProcessor::SPECIAL_PAGE );
128-
$result = $store->getQueryResult( $query ); // SMWQueryResult
129-
$wikiPageValues = $result->getResults(); // array of SMWWikiPageValues
160+
if ( $wgSDUUseJobQueue ) {
161+
$jobFactory = ApplicationFactory::getInstance()->newJobFactory();
130162

131-
return $wikiPageValues;
132-
}
163+
if ( $triggerSemanticDependencies ) {
133164

134-
/**
135-
* Rebuilds data of the given wikipages to regenerate semantic attrubutes and re-run queries
136-
*
137-
* @param SMWWikiPageValues[] $wikiPageValues
138-
* @param SMWStore $store
139-
*/
140-
public static function rebuildData( $wikiPageValues, $store ) {
141-
global $wgSDUUseJobQueue;
165+
$jobs = [];
142166

143-
$pageArray = [];
144167
foreach ( $wikiPageValues as $wikiPageValue ) {
145-
$page = WikiPage::newFromID( $wikiPageValue->getTitle()->getArticleId() );
146-
if ( $page ) {
147-
$pageArray[] = $page->getTitle()->prefixedText;
148-
}
168+
$jobs[] = $jobFactory->newUpdateJob(
169+
$wikiPageValue->getTitle(),
170+
[
171+
UpdateJob::FORCED_UPDATE => true,
172+
'shallowUpdate' => false
173+
]
174+
);
149175
}
150-
$pageString = implode( $pageArray, "|" );
151-
152-
// TODO: A threshold when to switch to Queue Jobs might be smarter
153-
154-
if ( $pageString !== "" ) {
155-
if ( $wgSDUUseJobQueue ) {
156-
$jobs = [];
157-
$jobs[] = new RebuildDataJob( [
158-
'pageString' => $pageString,
159-
] );
160-
foreach ( $wikiPageValues as $page ) {
161-
$jobs[] = new PageUpdaterJob( [
162-
'page' => $page
163-
] );
164-
}
165-
if ( $jobs ) {
166-
JobQueueGroup::singleton()->lazyPush( $jobs );
167-
}
168-
} else {
169-
DeferredUpdates::addCallableUpdate( static function () use ( $store, $pageString ) {
170-
wfDebugLog( 'SemanticDependencyUpdater', "[SDU] --------> [rebuildData] $pageString" );
171-
$maintenanceFactory = ApplicationFactory::getInstance()->newMaintenanceFactory();
172-
173-
$dataRebuilder = $maintenanceFactory->newDataRebuilder( $store );
174-
$dataRebuilder->setOptions(
175-
new Options( [ 'page' => $pageString ] )
176-
);
177-
$dataRebuilder->rebuild();
178-
} );
179-
}
176+
if ( $jobs ) {
177+
JobQueueGroup::singleton()->lazyPush( $jobs );
180178
}
179+
} else {
180+
DeferredUpdates::addCallableUpdate( static function () use ( $jobFactory, $wikiPageValues ) {
181+
$job = $jobFactory->newUpdateJob(
182+
$wikiPageValues[0]->getTitle(),
183+
[
184+
UpdateJob::FORCED_UPDATE => true,
185+
'shallowUpdate' => false
186+
]
187+
);
188+
$job->run();
189+
} );
190+
}
191+
181192
}
193+
}
182194

183195
}

Diff for: ‎src/PageUpdaterJob.php

-39
This file was deleted.

0 commit comments

Comments
 (0)