Add metrics tracking mechanism

index.js

@@ -8,18 +8,29 @@ const scraper = require('./playlist-scrape')
 const STREAM_URL = "https://www.youtube.com/henrikomagnifico/live"
 const DURATION_REGEX = /(\d{1,2}:\d{2})\/(\d{1,2}:\d{2})/
 const EXPIRATION_REGEX = /.+\/expire\/([0-9]{10})\/.+/
+const thresholds = [20, 17.5, 15, 12.5, 10, 7.5, 5].map(t => ({threshold: t, trackVotes: {correct: 0, skipped: 0}, positionVotes: {correct: 0, skipped: 0}}))
 let albumWhitelistCharacters = playlist.getValidAlbumCharacters()
 let trackWhitelistCharacters = playlist.getValidTrackCharacters()
 
 let currentTrack = {}
 let nextTrackTimestamp = 0
 let readAttempts = 0
+let totalVotes = 0
 
 let resolvedUrl = {
 	url: '',
 	expires: 0
 }
 
+var trackStatsFileStream = fs.createWriteStream("track.csv", {flags: 'a'});
+var positionStatsFileStream = fs.createWriteStream("position.csv", {flags: 'a'});
+thresholds.forEach(t => {
+	trackStatsFileStream.write(t.threshold + ",")
+	positionStatsFileStream.write(t.threshold + ",")
+})
+trackStatsFileStream.write('\n')
+positionStatsFileStream.write('\n')
+
 function readText(tmpfile, charWhitelist) {
 	return new Promise((resolve, reject) => {
 	    let command = `tesseract ${tmpfile.name} -`
@@ -32,7 +43,7 @@ function readText(tmpfile, charWhitelist) {
 				console.error(stderr)
 				reject(error)
 			} else {
-				resolve(stdout.replace('\n', '').replace('\f', ''))
+				resolve(stdout.replace(/[\n\f]/g, ''))
 			}
 		}))
 	})
@@ -100,7 +111,7 @@ function getYoutubeAudioUrl() {
 
 function getVotes(ocrResults) {
 	return ocrResults.map((text, index, arr) => { // Actually perform the search. Map returns array of search results, result.refIndex is of interest
-		console.debug(`Worker ${index} input:`, text)
+		new Promise(resolve => resolve()).then(console.debug(`Worker ${index} input: ${text}`))
 		const title = text[0].trim()
 		const album = text[1].trim()
 		const position_duration = text[2].trim()
@@ -121,34 +132,35 @@ function tallyVotes(votes) {
 	let positionVotes = 0
 	const voters = []
 	const durationVotes = []
+	totalVotes++
 	votes.forEach((vote, index, arr) => {
 		if (vote.result != null) {
 			overallVotes++
-			voters.push("v")
+			voters.push("y")
 			if (trackResults.hasOwnProperty(vote.result.refIndex)) {
 				trackResults[`${vote.result.refIndex}`] = trackResults[`${vote.result.refIndex}`] + 1
 			} else {
 				trackResults[`${vote.result.refIndex}`] = 1
 			}
 		} else {
-			voters.push("x")
+			voters.push("n")
 		}
 		if (vote.position) {
 			positionVotes++
-			durationVotes.push("v")
+			durationVotes.push("y")
 			if (positionResults.hasOwnProperty(vote.position)) {
 				positionResults[`${vote.position}`] = positionResults[`${vote.position}`] + 1
 			} else {
 				positionResults[`${vote.position}`] = 1
 			}
 		} else {
-			durationVotes.push("x")
+			durationVotes.push("n")
 		}
 	})
 
-	console.debug("Voter status:")
-	console.debug("Result vote:", voters)
-	console.debug("Duration vote:", durationVotes)
+	console.debug("\nDid workers vote?")
+	console.debug("Result vote:", JSON.stringify(voters))
+	console.debug("Duration vote:", JSON.stringify(durationVotes))
 	if (overallVotes === 0) {
 		console.warn("no workers voted")
 		return {
@@ -174,7 +186,7 @@ function tallyVotes(votes) {
 	let trackMaxVotes = trackResults[trackIndex]
 	let position = Object.keys(positionResults)[0]
 	let positionMaxVotes = positionResults[position]
-	console.debug("Initial index:", trackIndex, trackResults)
+	console.debug("Initial index:", trackIndex, "All vote results:", trackResults)
 	for (const index in trackResults) {
 		if (trackResults[index] > trackMaxVotes) {
 			trackIndex = index
@@ -189,13 +201,40 @@ function tallyVotes(votes) {
 	}
 
 	// console.log(JSON.stringify(votes))
-	console.debug(`Vote was index`, trackIndex)
-	console.debug("Voter results:")
-	// console.debug("Track accuracy:", voters.map((v, i, a) => {
-		// if (v === 'v') {
-			
-		// }
-	// }))
+	console.debug(`\nVote was index`, trackIndex)
+	console.debug("Worker results:")
+    const voterTrackAccuracy = voters.map((v, i, a) => {
+		if (v === 'y') {
+			const voteWasCorrect = `${votes[i].result.refIndex}` === trackIndex
+			if (voteWasCorrect) {
+				thresholds[i].trackVotes.correct++
+				return 'Y'
+			} else {
+				return 'n'
+			}
+		} else {
+			thresholds[i].trackVotes.skipped++
+			return '_'
+		}
+	})
+	const voterPositionAccuracy = durationVotes.map(((value, index, array) => {
+		if (value === 'y') {
+			const voteWasCorrect = votes[index].position === position
+            if (voteWasCorrect) {
+            	thresholds[index].positionVotes.correct++
+				return 'Y'
+			} else {
+            	return 'n'
+			}
+		} else {
+			thresholds[index].positionVotes.skipped++
+			return '_'
+		}
+	}))
+	console.debug("Track accuracy:", JSON.stringify(voterTrackAccuracy))
+	trackStatsFileStream.write(voterTrackAccuracy.join(','))
+	console.debug("Position accuracy:", JSON.stringify(voterPositionAccuracy))
+	positionStatsFileStream.write(voterPositionAccuracy.join(','))
 	const actualResult = votes.find(v => v.result != null && `${v.result.refIndex}` === trackIndex).result.item
 	return {
 		position: position,
@@ -219,12 +258,11 @@ function performQuorumProcessing(titleCroppedPromise, albumCroppedPromise, durat
 	return new Promise(((resolve, reject) => {
 		Promise.all([titleCroppedPromise, albumCroppedPromise, durationCroppedPromise]) // Wait for all promises to be resolved, returns paths to cropped segments of frame
 			.then(images => {
-				const thresholds = [20, 17.5, 15, 12.5, 10, 7.5, 5]
 				Promise.all(thresholds.map(t => { // Threshold all segments at determined levels, returns text determined by each threshold level as [title, album, position/duration]
 					return Promise.all([
-						thresholdImage(images[0], t).then(file => readText(file)),
-						thresholdImage(images[1], t).then(file => readText(file)),
-						thresholdImage(images[2], t).then(file => readText(file, "1234567890:/"))
+						thresholdImage(images[0], t.threshold).then(file => readText(file)),
+						thresholdImage(images[1], t.threshold).then(file => readText(file)),
+						thresholdImage(images[2], t.threshold).then(file => readText(file, "1234567890:/"))
 					])
 				})).then(allOcrResults => { // Perform search based on resolved text. Perform voting here between thresholds
 				    // console.debug(allOcrResults)
@@ -235,7 +273,7 @@ function performQuorumProcessing(titleCroppedPromise, albumCroppedPromise, durat
 					// Tally votes
 					const winner = tallyVotes(votes)
 
-					console.debug('Position confidence:', winner.positionConfidence)
+					console.debug('\nPosition confidence:', winner.positionConfidence)
 					console.debug('Result confidence:', winner.resultConfidence)
 					console.debug('Voter confidence:', winner.voterConfidence)
 
@@ -260,28 +298,49 @@ async function updateTrackInfo() {
 		updateStreamUrl().then(() => console.log("Updated stream info in the background"))
 	}
 	console.log(`Now: ${Date.now()}\tExpiration: ${resolvedUrl.expires}\tAttempt: ${readAttempts}`)
+	const processingStart = Date.now()
 	const frame = await getFrame(resolvedUrl.url)
     const frameTime = Date.now()
 
-	const titlePromise = getRegion(frame, 432, 906, 1487, 54)
+	const titlePromise = getRegion(frame, 432, 850, 1487, 100)
 	const albumPromise = getRegion(frame, 440, 957, 1487, 32)
 	const durationPromise = getRegion(frame, 0, 1028, 235, 34)
 
 	const result = await performQuorumProcessing(titlePromise, albumPromise, durationPromise)
-	if (result.resultConfidence < 0.4) {
+	trackStatsFileStream.write(',' + readAttempts + '\n')
+	const votingTime = Date.now()
+	if (result.resultConfidence < 0.5) {
+		console.debug("Result confidence not high enough, retrying")
 		setTimeout(() => updateTrackInfo(), 0)
 		readAttempts++
 		return;
 	}
 	readAttempts = 0
 	currentTrack = result.result
-	console.log(`${currentTrack.album}: ${currentTrack.track} (${result.position}/${currentTrack.duration})`)
-	const secondsUntilNext = timeUntilNextTrack(result.position, currentTrack.duration)
-	const processingTime = Date.now() - frameTime
-	nextTrackTimestamp = frameTime + (secondsUntilNext * 1000)
-	setTimeout(() => updateTrackInfo(), (secondsUntilNext * 1000) - processingTime + 250)
-	console.log(`Frame processing took ${processingTime}ms`)
-	console.log(`Next track should be at: ${new Date(nextTrackTimestamp).toLocaleString()}`)
+	console.debug("\nCurrent threshold statistics: ")
+	thresholds.forEach(t => {
+		const skippedTrackRatio = (t.trackVotes.skipped / totalVotes) * 100
+		const skippedPositionRatio = (t.positionVotes.skipped / totalVotes) * 100
+		const accurateTrackRatio = (t.trackVotes.correct / (totalVotes - t.trackVotes.skipped)) * 100
+		const accuratePositionRatio = (t.positionVotes.correct / (totalVotes - t.positionVotes.skipped)) * 100
+		const trackEffectiveness = accurateTrackRatio - skippedTrackRatio
+		const positionEffectiveness = accuratePositionRatio - skippedPositionRatio
+		console.log(`${t.threshold}%:\tTrack: [Skip: ${skippedTrackRatio.toFixed(2)}% Correct: ${accurateTrackRatio.toFixed(2)}% Eff.: ${trackEffectiveness.toFixed(2)}%]\tPos: [Skip: ${skippedPositionRatio.toFixed(2)}% Correct: ${accuratePositionRatio.toFixed(2)}% Eff.: ${positionEffectiveness.toFixed(2)}%]`)
+	})
+	if (result.position) {
+		const secondsUntilNext = timeUntilNextTrack(result.position, currentTrack.duration)
+		const processingTime = Date.now() - processingStart
+		const timeInFrame = Date.now() - frameTime
+		nextTrackTimestamp = frameTime + (secondsUntilNext * 1000)
+		setTimeout(() => updateTrackInfo(), (secondsUntilNext * 1000) - timeInFrame + 750)
+		console.log(`\nFrame processing took ${processingTime}ms (Frame retrieval: ${frameTime - processingStart}ms) (Processing & voting: ${votingTime - frameTime}ms)`)
+		console.log(`${currentTrack.album}: ${currentTrack.track} (${result.position}/${currentTrack.duration})`)
+		console.log(`Next track should be at: ${new Date(nextTrackTimestamp).toLocaleString()}`)
+	} else {
+		console.log(`${currentTrack.album}: ${currentTrack.track}`)
+		console.warn("Workers did not vote on current position, retrying...")
+		setTimeout(() => updateTrackInfo(), 0)
+	}
 }
 
 // getYoutubeStream()

playlist-search.js

@@ -80,7 +80,7 @@ function filterResultsByDuration(results, duration, failSearch = false) {
 }
 
 function search(albumQuery, trackQuery, duration) {
-	if (albumQuery && trackQuery && duration) {
+	if (albumQuery && trackQuery) {
 		return fullSearch(albumQuery, trackQuery, duration)
 	} else if (albumQuery && duration) {
 		return albumDurationSearch(albumQuery, duration)
@@ -88,6 +88,7 @@ function search(albumQuery, trackQuery, duration) {
 		return trackDurationSearch(trackQuery, duration)
 	} else {
 	    // Not enough info to perform a decent search
+		// console.log("Skipping search for", albumQuery, trackQuery, duration)
 		return null
 	}
 }