User:DannyS712/SortedPropertiesUpdater.js

// <nowiki>
// Quick script to Simplify updating MediaWiki:Wikibase-SortedProperties
// @author DannyS712
/* jshint maxerr: 999 */
$(() => {
const SortedPropertiesUpdater = {};
window.SortedPropertiesUpdater = SortedPropertiesUpdater;

// Included in the edit summary
SortedPropertiesUpdater.version = '1.2';

SortedPropertiesUpdater.init = function () {
	window.document.title = 'SortedProperties updater';
	$( '#firstHeading' ).text( 'SortedProperties updater' );
	mw.loader.using(
		[ 'oojs-ui-core', 'oojs-ui-widgets', 'oojs-ui-windows', 'mediawiki.api', 'mediawiki.util' ],
		SortedPropertiesUpdater.run
	);
};

SortedPropertiesUpdater.onErrHandler = function () {
	// Shared error handler
	alert( 'Something went wrong' );
	console.log( arguments );
};

SortedPropertiesUpdater.startedProcessing = false;

// Map technical name to that for the section `Other properties with datatype "?"`
// Key: string data type from api (was stored in propertyDataTypesCache)
// Value: string data type for section name
SortedPropertiesUpdater.datatypeMap = {
	'commonsMedia': 'Commons media file',
	// Omitting external id intentionally, handled separately
	'geo-shape': 'geo-shape',
	'globe-coordinate': 'geographic coordinates',
	'math': 'math',
	'monolingualtext': 'monolingual text',
	'musical-notation': 'musical-notation',
	'quantity': 'quantity',
	'string': 'string',
	'tabular-data': 'tabular data',
	'time': 'point in time',
	'url': 'url',
	'wikibase-form': 'wikibase-form',
	'wikibase-item': 'item',
	'wikibase-lexeme': 'wikibase-lexeme',
	'wikibase-property': 'wikibase-property',
	'wikibase-sense': 'wikibase-sense'
};

SortedPropertiesUpdater.run = function () {
	var latestPropertyWidget = new OO.ui.TextInputWidget();
	var latestPropertyLayout = new OO.ui.FieldLayout(
		latestPropertyWidget,
		{ label: 'Latest property (in the form `xxx` for Pxxx)' }
	);
	var autoUpdateWidget = new OO.ui.CheckboxInputWidget();
	var autoUpdateLayout = new OO.ui.FieldLayout(
		autoUpdateWidget,
		{ label: 'Automatically update (BE CAREFUL AND DOUBLE CHECK)' }
	);
	var submit = new OO.ui.ButtonInputWidget( { 
		label: 'View updates',
		flags: [ 'primary', 'progressive' ]
	} );
	submit.on( 'click', function () {
		// CONVERT YOUR WIDGETS TO INPUT
		var submittedValues = {
			latestProperty: latestPropertyWidget.getValue(),
			autoUpdate: autoUpdateWidget.isSelected()
		};
		console.log( submittedValues );
		SortedPropertiesUpdater.onSubmit( submittedValues );
	} );
	$( window ).on( 'keypress', function ( e ) {
		// press enter to start
		if ( e.which == 13 ) {
			submit.simulateLabelClick();
		}
	} );
	
	var fieldSet = new OO.ui.FieldsetLayout( { 
		label: $( '<span>' )
			.append( 'View and implement updates needed for ' )
			.append(
				$( '<a>' )
					.attr( 'href', '/wiki/MediaWiki:Wikibase-SortedProperties' )
					.attr( 'target', '_blank' )
					.text( '[[MediaWiki:Wikibase-SortedProperties]]' ),
				$( '<small>' ).append(
					' (',
					$( '<a>' )
						.attr( 'href', '/wiki/MediaWiki:Wikibase-SortedProperties?diff=cur&oldid=prev&diffonly=true' )
						.attr( 'target', '_blank' )
						.text( 'last edit' ),
					') (',
					$( '<a>' )
						.attr( 'href', '/wiki/User:DannyS712/SortedPropertiesUpdater.js' )
						.attr( 'target', '_blank' )
						.text( '[[User:DannyS712/SortedPropertiesUpdater.js]]' ),
					')'
				).css( 'font-weight', 'normal' )
			)
	} );
	
	fieldSet.addItems( [
		latestPropertyLayout,
		autoUpdateLayout,
		new OO.ui.FieldLayout( submit )
	] );

	var $results = $( '<div>' )
		.attr( 'id', 'SortedPropertiesUpdater-results' );
	$( '#mw-content-text' ).empty().append(
		fieldSet.$element,
		$( '<hr>' ),
		$results
	);
	
	SortedPropertiesUpdater.autoFillLatestProperty( latestPropertyWidget );
};

SortedPropertiesUpdater.autoFillLatestProperty = function ( inputWidget ) {
	SortedPropertiesUpdater.getLatestProperty().then(
		function ( latestProperty ) {
			console.log( 'Got latest property: ' + latestProperty );
			if ( SortedPropertiesUpdater.startedProcessing === false ) {
				inputWidget.setValue( latestProperty );
				console.log( 'Updated widget accordingly' );
			}
		}
	);
};
SortedPropertiesUpdater.getLatestProperty = function () {
	return new Promise(
		function ( resolve ) {
			new mw.Api().get( {
				action: 'query',
				format: 'json',
				formatversion: 2,
				list: 'recentchanges',
				rcdir: 'older',
				rcnamespace: 120, // property
				rcprop: 'title',
				rclimit: 1, // just the latest
				rctype: 'new' // looking for new property creations
			} ).then(
				function ( response ) {
					console.log( response );
					var propertyTitle = response.query.recentchanges[0].title;
					var propertyId = propertyTitle.replace( 'Property:P', '' );
					resolve( propertyId );
				},
				SortedPropertiesUpdater.onErrHandler
			);
		}
	);
};

// Calls itself recursively until everything is retrieved
SortedPropertiesUpdater.getRawLabels = function ( propertyIds ) {
	// TODO handle non-admins running this script with 50 instead of 500
	return new Promise(
		function ( resolve ) {
			new mw.Api().get( {
				action: 'wbgetentities',
				formatversion: 2,
				ids: propertyIds.slice( 0, 500 ),
				languages: 'en',
				props: 'labels'
			} ).then(
				function ( response ) {
					console.log( response );
					var labels = response.entities;
					if ( propertyIds.length > 500 ) {
						// recursive
						SortedPropertiesUpdater.getRawLabels( propertyIds.slice( 500 ) ).then(
							function ( extraLabels ) {
								var bothLabels = $.extend( {}, labels, extraLabels );
								resolve( bothLabels );
							},
							SortedPropertiesUpdater.onErrHandler
						);
					} else {
						resolve( labels );
					}
				},
				SortedPropertiesUpdater.onErrHandler
			);
		}
	);
};

SortedPropertiesUpdater.propertyDataTypesCache = {};
SortedPropertiesUpdater.getRowsForProperties = function ( labelInfo ) {
	var properties = Object.keys( labelInfo );
	var rows = [];
	
	var propertyInfo;
	var propertyRow;
	properties.forEach( function ( propertyId ) {
		propertyInfo = labelInfo[ propertyId ];
		if ( propertyInfo.missing !== undefined ) {
			return;
		}
		propertyRow = propertyInfo.id + ' (' + propertyInfo.labels.en.value + ')';
		rows.push( propertyRow );
		
		// For later
		SortedPropertiesUpdater.propertyDataTypesCache[ propertyInfo.id ] = propertyInfo.datatype;
	} );
	console.log( rows );
	return rows;
};

SortedPropertiesUpdater.currentTextCache = false;

SortedPropertiesUpdater.getCurrentSortedProperties = function () {
	return new Promise(
		function ( resolve ) {
			new mw.Api().get( {
				action: 'query',
				prop: 'revisions',
				rvprop: 'content',
				rvslots: 'main',
				titles: 'MediaWiki:Wikibase-SortedProperties',
				formatversion: 2
			} ).then(
				function ( response ) {
					console.log( response );
					var text = response.query.pages[0].revisions[0].slots.main.content;
					SortedPropertiesUpdater.currentTextCache = text; // for use later
					var rows = text.split( '\n' );
					rows = rows.filter(
						function ( row ) {
							// Ignore the section headings, etc.
							// Some entries start with *, others with #
							return ( row.startsWith( '* P' ) || row.startsWith( '# P' ) );
						}
					);
					rows = rows.map(
						function ( row ) {
							// remove the first two characters, the * or # and the space
							return row.substring( 2 );
						}
					);
					resolve( rows );
				},
				SortedPropertiesUpdater.onErrHandler
			);
		}
	);
};

SortedPropertiesUpdater.compareWithCurrent = function ( goodRows, autoUpdate ) {
	SortedPropertiesUpdater.getCurrentSortedProperties().then( function( currentSortedProperties ) {
		console.log( currentSortedProperties );
		
		// Handle changes in labels first
		// Map of property id to label
		var currentLabels = {}, goodLabels = {};
		var rowInfo;
		currentSortedProperties.forEach(
			function ( currentRow ) {
				rowInfo = currentRow.match( /^(P\d+) \((.*?)\)$/ );
				if ( rowInfo ) {
					currentLabels[ rowInfo[1] ] = rowInfo[ 2 ];
				} else {
					console.log( 'Odd row in currentSortedProperties: ' + currentRow );
				}
			}
		);
		goodRows.forEach(
			function ( goodRow ) {
				rowInfo = goodRow.match( /^(P\d+) \((.*?)\)$/ );
				if ( rowInfo ) {
					goodLabels[ rowInfo[1] ] = rowInfo[ 2 ];
				} else {
					console.log( 'Odd row in goodRows: ' + goodRow );
				}
			}
		);
		
		var labelChanges = [];
		var labelChangesData = {};
		var rowsToRemove = [];
		var rowsToAdd = [];
		var rowsToAddData = {};
		var currentLabelForProperty;
		Object.keys( currentLabels ).forEach(
			function ( propertyId ) {
				currentLabelForProperty = currentLabels[ propertyId ];
				if ( goodLabels[ propertyId ] === undefined ) {
					// No good label for this property, so it was deleted
					rowsToRemove.push( propertyId + ' (' + currentLabelForProperty + ')' );
					return;
				}
				
				// There is a label for this property, compare it
				if ( goodLabels[ propertyId ] !== currentLabelForProperty ) {
					labelChangesData[ propertyId ] = [ currentLabelForProperty, goodLabels[ propertyId ] ];
					labelChanges.push( propertyId + ': ' + currentLabelForProperty + ' => ' + goodLabels[ propertyId ] );
				}
			}
		);
		
		Object.keys( goodLabels ).forEach(
			function ( propertyId ) {
				// We have a good label. If there is no prior label, it means its a new property to add
				if ( currentLabels[ propertyId ] === undefined ) {
					rowsToAdd.push( propertyId + ' (' + goodLabels[ propertyId ] + ')' );
					rowsToAddData[ propertyId ] = goodLabels[ propertyId ];
				}
			}
		);
		
		var sortedRowsToAdd = SortedPropertiesUpdater.formatRowsToAdd( rowsToAddData );
		var $res = $( '#SortedPropertiesUpdater-results' );
		$res.append(
			$( '<p>' ).append( 'Checked against current rows. Results:' )
		);
		$res.append(
			$( '<pre>' ).text(
				'Rows to change:\n' + labelChanges.join( '\n' ) +
				'\n\n' +
				'Rows to remove:\n' + rowsToRemove.join( '\n' ) +
				'\n\n' +
				'Rows to add:\n' + rowsToAdd.join( '\n' ) +
				'\n\n' +
				'Sorted rows to add info:\n' + sortedRowsToAdd
			)
		);
		
		SortedPropertiesUpdater.makeNewContent( labelChangesData, rowsToAddData, rowsToRemove, autoUpdate );
	} );
};

SortedPropertiesUpdater.formatRowsToAdd = function ( rowsToAddData ) {
	var resultText = '';
	if ( rowsToAddData ) {
		// object with keys as property ids, values as the labels
		// use SortedPropertiesUpdater.propertyDataTypesCache
		var propertiesToAddByType = {};
		Object.keys( rowsToAddData ).forEach(
			function ( propertyIdToAdd ) {
				var propertyType = SortedPropertiesUpdater.propertyDataTypesCache[ propertyIdToAdd ];
				if ( propertiesToAddByType[ propertyType ] === undefined ) {
					propertiesToAddByType[ propertyType ] = [];
				}
				propertiesToAddByType[ propertyType ].push(
					'* ' + propertyIdToAdd + ' (' + rowsToAddData[ propertyIdToAdd ] + ')'
				);
			}
		);
		Object.keys( propertiesToAddByType ).forEach(
			function ( dataType ) {
				resultText = resultText + '; Data type ' + dataType + '\n' + propertiesToAddByType[ dataType ].join( '\n' ) + '\n';
			}
		);
	}
	return resultText;
};

SortedPropertiesUpdater.makeNewContent = function ( labelChangesData, rowsToAddData, rowsToRemove, autoUpdate ) {
	var newText = SortedPropertiesUpdater.currentTextCache;
	var replacementRegex;
	var replacementInfo;
	Object.keys( labelChangesData ).forEach(
		function ( propertyNeedingLabelChange ) {
			// key is the property id, value is an array of the current label and the new label
			replacementInfo = labelChangesData[ propertyNeedingLabelChange ];
			replacementRegex = new RegExp(
				propertyNeedingLabelChange +
				' \\(' + mw.util.escapeRegExp( replacementInfo[ 0 ] ) + '\\)\\n'
			);
			newText = newText.replace(
				replacementRegex,
				propertyNeedingLabelChange + ' (' + replacementInfo[ 1 ] + ')\n'
			);
		}
	);
	
	if ( Object.keys( rowsToAddData ).length > 0 ) {
		// filter out new external ids, those are sorted into alphabetical order section seperately
		var newExternalIds = [];
		Object.keys( rowsToAddData ).forEach(
			function ( propertyIdToAdd ) {
				var propertyType = SortedPropertiesUpdater.propertyDataTypesCache[ propertyIdToAdd ];
				if ( propertyType === 'external-id' ) {
					// add the # here so that we can be consistent in the handling with the existing entries
					newExternalIds.push( '# ' + propertyIdToAdd + ' (' + rowsToAddData[ propertyIdToAdd ] + ')' );
					delete rowsToAddData[ propertyIdToAdd ];
				}
			}
		);
		console.log( newExternalIds );
		// retrieve current external ids in alphabetical order
		var currentExternalIdsABCmatch = newText.match(
			/<!--External id alphabetical order start-->\n([\s\S]+?)\n<!--External id alphabetical order end-->/
		);
		if ( currentExternalIdsABCmatch && currentExternalIdsABCmatch[1] ) {
			var currentExternalIdsABC = currentExternalIdsABCmatch[1];
			var unsortedExternalIds = currentExternalIdsABC.split( '\n' );
			unsortedExternalIds = unsortedExternalIds.concat( newExternalIds );
			unsortedExternalIds.sort(
				function ( row1, row2 ) {
					row1Info = row1.match( /^# P\d+ \((.*?)\)$/ );
					row2Info = row2.match( /^# P\d+ \((.*?)\)$/ );
					if ( !row1Info || !row2Info ) {
						return 0;
					}
					return row1Info[1].localeCompare( row2Info[1] );
				}
			);
			var updatedSortedExternalIds = unsortedExternalIds.join( '\n' );
			newText = newText.replace(
				/<!--External id alphabetical order start-->\n([\s\S]+?)\n<!--External id alphabetical order end-->/,
				'<!--External id alphabetical order start-->\n' + updatedSortedExternalIds + '\n<!--External id alphabetical order end-->'
			);
		}
		
		if ( Object.keys( rowsToAddData ).length > 0 ) {
			// need check rowsToAddData again since maybe all of the new properties were external ids handled separately
			// Try to add to relevant sections by datatype
			Object.keys( rowsToAddData ).forEach(
				function ( propertyIdToAdd ) {
					var propertyType = SortedPropertiesUpdater.propertyDataTypesCache[ propertyIdToAdd ];
					var dataTypeInSection = SortedPropertiesUpdater.datatypeMap[ propertyType ];
					if ( !dataTypeInSection ) {
						// Anything I missed, or new properties being added...
						return;
					}
					var sectionRegex = new RegExp(
						'(=== Other properties with datatype "' + dataTypeInSection + '" ===(?:\\n\\* P\\d+ .*)+)'
					);
					var sectionMatch = newText.match( sectionRegex );
					if ( !sectionMatch || !sectionMatch[1] ) {
						// Missing section, or I messed up
						return;
					}
					// Add entry to relevant section
					newText = newText.replace(
						sectionRegex,
						sectionMatch[1] + '\n* ' + propertyIdToAdd + ' (' + rowsToAddData[ propertyIdToAdd ] + ')'
					);
					// Remove from unsorted
					delete rowsToAddData[ propertyIdToAdd ];
				}
			);
		}
		
		if ( Object.keys( rowsToAddData ).length > 0 ) {
			// need check rowsToAddData again since maybe all of the new properties were external ids handled separately, or
			// were properly sorted by section
			
			// the "Unsorted properties" heading might already be there, if so don't add another
			if ( !( newText.match( /==\s*Unsorted properties\s*==/i ) ) ) {
				// only 1 newline here, so that we can ensure there is always a newline before the new entries if there
				// was a heading already
				newText = newText + '\n\n== Unsorted properties ==\n';
			}
			newText = newText + '\n' + SortedPropertiesUpdater.formatRowsToAdd( rowsToAddData );
		}
	}

	// Automatically remove deleted properties
	var removalRegex;
	rowsToRemove.forEach(
		function ( deletedProperty ) {
			removalRegex = new RegExp(
				'\\n[*#] ' + mw.util.escapeRegExp( deletedProperty )
			);
			newText = newText.replace(
				removalRegex,
				""
			);
		}
	);
	
	$( '#SortedPropertiesUpdater-results' ).append(
		$( '<p>' ).append( 'An updated text is available to copy from the console' )
	);
	console.log( newText );
	
	if ( autoUpdate ) {
		SortedPropertiesUpdater.autoUpdate( newText );
	}
};

SortedPropertiesUpdater.autoUpdate = function ( newText ) {
	mw.notify( 'Automatically updating!', { autoHide: false } );
	var params = {
		action: 'edit',
		title: 'MediaWiki:Wikibase-SortedProperties',
		text: newText,
		summary: 'Automatically updating via [[User:DannyS712/SortedPropertiesUpdater.js]] (version ' + SortedPropertiesUpdater.version + ')',
		nocreate: true, // just in case
		assert: 'user', // non users shouldn't be able to edit MediaWiki namespace, but why not?
		format: 'json',
		formatversion: 2
	};
	console.log( params );
	new mw.Api().postWithEditToken( params ).then(
		function ( response ) {
			console.log( response );
			mw.notify( 'Finished auto update', { autoHide: false } );
		},
		SortedPropertiesUpdater.onErrHandler
	);
};

SortedPropertiesUpdater.onSubmit = function ( inputs ) {
	SortedPropertiesUpdater.startedProcessing = true;

	var $res = $( '#SortedPropertiesUpdater-results' );
	$res.empty();
	
	console.log( inputs );
	var latestProperty = inputs.latestProperty;
	
	$res.append(
		$( '<p>' ).append(
			'Checking the api for labels of properties up to P' + latestProperty
		)
	);
	
	// Array from P1 to P<latestProperty>
	var propertyIds = [];
	for ( var iii = 1; iii <= latestProperty; iii++ ) {
		propertyIds.push( 'P' + iii );
	}
	SortedPropertiesUpdater.getRawLabels( propertyIds ).then(
		function ( labelInfo ) {
			console.log( labelInfo );
			$res.append(
				$( '<p>' ).append( 'Api calls done, now converting to rows' )
			);
			var goodRows = SortedPropertiesUpdater.getRowsForProperties( labelInfo );
			$res.append(
				$( '<p>' ).append( '...now comparing to current content' )
			);
			SortedPropertiesUpdater.compareWithCurrent(
				goodRows,
				inputs.autoUpdate
			);
		}
	);
};

});

$( document ).ready( () => {
	mw.loader.using(
		[ 'mediawiki.util' ],
		function () {
			mw.util.addPortletLink(
				'p-tb',
				'/wiki/Special:BlankPage/SortedPropertiesUpdater',
				'View updates for SortedProperties'
			);
		}
	);
	if ( mw.config.get( 'wgNamespaceNumber' ) === -1 ) {
		const page = mw.config.get( 'wgCanonicalSpecialPageName' );
		if ( page === 'Blankpage' ) {
			const page2 = mw.config.get( 'wgTitle' ).split( '/' );
			if ( page2[1] && page2[1] === 'SortedPropertiesUpdater' ) {
				window.SortedPropertiesUpdater.init();
			}
		}
	}
});

// </nowiki>