Add music transfer script munging tests

This is primarily to ensure emojis are replaced as file names can't
contain those when transferred via go-mtpfs. This has the side effect of
encoding _all_ unicode characters, such as japanese characters, and
other emoticons, but that's a small price to pay for such an easy fix.

src/.bashrc

@@ -114,6 +114,7 @@ alias isodate='date -I'
 alias isotime='date -Ihours'
 alias headers='curl -I'              # get site headers
 alias headersc='curl -I --compress'  # test gzip/mod_deflate support
+alias auu='sudo apt update && sudo apt upgrade'
 
 # add colors for filetype and human-readable sizes in `ls`
 alias ls='ls -h --color --show-control-chars'

src/.bin/music-transfer.py

@@ -1,158 +0,0 @@
-#!/bin/python3
-
-###
-### Music transfer script useful for syncing a subset of music from a source
-### (home computer, for example) to a portable device, converting files to mp3
-### for portability.
-###
-
-import os
-from pathlib import Path
-import re
-import shutil
-import subprocess
-import sys
-from typing import (
-	Set,
-	Union,
-)
-
-KNOWN_EXTENSIONS: Set[str] = set([
-	'.mp3',
-	'.flac',
-	'.m4a',
-	'.opus',
-])
-
-# TODO: improve copying from ext4 to fs which can't handle ':' in paths
-#       that is, don't munge all the possibly problematic chars, only do
-#       the ones which cause problems on the target fs
-# see: https://stackoverflow.com/a/35352640/540162
-# getting filesystem type can be done like so: https://stackoverflow.com/a/25286268/540162
-def munge_path(path: str) -> str:
-	return re.sub(r'[\:*?"|<>]+', '-', path)
-
-if len(sys.argv) != 3:
-	print(f'Usage: {os.path.basename(__file__)} PATHS-FILE DESTINATION-DIR', file=sys.stderr)
-	sys.exit(1)
-
-paths_file: str = sys.argv[1]
-destination_dir: str = sys.argv[2]
-
-if not os.path.exists(paths_file):
-	print(f'PATHS-FILE \'{paths_file}\' does not exist or is not readable', file=sys.stderr)
-	sys.exit(2)
-
-
-print('[DEBUG] Gathering destination files list')
-destination_files: Set[Path] = set()
-if os.path.isdir(destination_dir):
-	for dirpath, _, files in os.walk(destination_dir):
-		for file in files:
-			path = Path(os.path.join(dirpath, file)).absolute()
-			if path.suffix == '.mp3':
-				if path in destination_files:
-					print(f'Ignoring duplicate input entry: {file}')
-				else:
-					destination_files.add(path)
-print(f'[DEBUG] Gathered {len(destination_files)} destination files')
-
-print('[DEBUG] Gathering source files list')
-source_files: Set[Path] = set()
-for line in open(paths_file, 'r'):
-	trimmed_line = line.strip()
-	if not trimmed_line:
-		continue
-
-	path = Path(trimmed_line).absolute()
-	if path.is_file():
-		if path.suffix in KNOWN_EXTENSIONS:
-			source_files.add(path)
-		else:
-			print(f'[TRACE] Unknown file extension of entry: {trimmed_line}', file=sys.stderr)
-	elif path.is_dir():
-		for dirpath, _, files in os.walk(path.absolute()):
-			# TODO: DRY (see above)
-			for file in files:
-				file_path = Path(os.path.join(dirpath, file)).absolute()
-				if file_path.suffix in KNOWN_EXTENSIONS:
-					source_files.add(file_path)
-				else:
-					print(f'[TRACE] Unknown file extension of entry: {file}', file=sys.stderr)
-	elif not path.exists():
-		print(f'[ERROR] Could not find source path: {path}')
-		sys.exit(3)
-
-if len(source_files) == 0:
-	print('No source files to transfer')
-	sys.exit()
-else:
-    print(f'[DEBUG] Gathered {len(source_files)} source files')
-
-print('[DEBUG] Finding longest common prefix of source files')
-longest_prefix = next(iter(source_files)).parent
-for file in source_files:
-	if longest_prefix.as_posix() == '/':
-		break
-
-	while not file.as_posix().startswith(longest_prefix.as_posix()):
-		longest_prefix = longest_prefix.parent
-
-print('[DEBUG] Filtering files already present in destination')
-# NOTE: this assumes all filenames are unique, which should already be the case in my music library
-#       verify via `find . -type f -not -name '*.jpg' -not -name '*.png' -not -name '*.txt' -exec basename {} \; | sort | uniq -d`
-# TODO: add a hash check so that we can overwrite songs on the target if source
-#       has a version which is different (new metadata, or better version)
-for source_path in list(source_files)[:]:
-	found_destination_path: Union[Path, None] = next((destination_file for destination_file in destination_files if destination_file.stem == munge_path(source_path.stem)), None)
-	if found_destination_path:
-		source_files.remove(source_path)
-		destination_files.remove(found_destination_path)
-
-if len(source_files) == 0:
-	print('All source files already exist in target')
-	sys.exit()
-
-# TODO: improve prompt (print 10+ file paths or exit to `PAGER` to view before prompt so user has more info before confirming)
-if len(destination_files) > 0:
-	delete_selection_made = False
-	while not delete_selection_made:
-		print(f'Files in destination not found in source files list:\n{destination_files}')
-		delete_input = input(f'Delete {len(destination_files)} files not found in source files list? [y/n] ')
-		if delete_input[0] == 'y' or delete_input[0] == 'Y':
-			delete_selection_made = True
-			for destination_file in destination_files:
-				destination_file.unlink()
-		elif delete_input[0] == 'n' or delete_input[0] == 'N':
-			delete_selection_made = True
-
-# TODO: list both number of files in songs list & number of files to be transferred
-print(f'Copying {len(source_files)} songs to {destination_dir}')
-print(f'[TRACE] Copying the following files to destination:\n{source_files}')
-
-for source_file in source_files:
-	source_copy_path = source_file.with_suffix('').as_posix().replace(longest_prefix.as_posix(), '', 1)
-	destination_filename = munge_path(source_copy_path)
-	destination_file_path = f'{destination_dir}/{destination_filename}.mp3'
-
-	# TODO: double-check on this
-	print(f'destination_file_path: {destination_file_path}')
-	Path(destination_file_path).parent.mkdir(parents = True, exist_ok = True)
-
-	if source_file.stem == '.mp3':
-		shutil.copy2(source_file.as_posix(), destination_file_path)
-	else:
-		subprocess.call([
-			'ffmpeg',
-			'-loglevel', 'quiet',
-			'-i', source_file.as_posix(),
-			'-codec:a', 'libmp3lame',
-			# convert to mp3 with quality level 3 vbr (average 175 kbit/s, ranges from 150-195 kbit/s)
-			# see: https://trac.ffmpeg.org/wiki/Encode/MP3#VBREncoding
-			'-q:a', '3',
-			destination_file_path
-		])
-
-
-# TODO: improve this output (number of deleted files, number of new files, correct plural of 'files')
-print(f'Finished copying {len(source_files)} new files to {destination_dir}')

src/.bin/music_transfer.py

@@ -0,0 +1,162 @@
+#!/bin/python3
+
+###
+### Music transfer script useful for syncing a subset of music from a source
+### (home computer, for example) to a portable device, converting files to mp3
+### for portability.
+###
+
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+from typing import (
+	Set,
+	Union,
+)
+
+KNOWN_EXTENSIONS: Set[str] = set([
+	'.mp3',
+	'.flac',
+	'.m4a',
+	'.opus',
+])
+
+# TODO: improve copying from ext4 to fs which can't handle ':' in paths
+#       that is, don't munge all the possibly problematic chars, only do
+#       the ones which cause problems on the target fs
+# see: https://stackoverflow.com/a/35352640/540162
+# getting filesystem type can be done like so: https://stackoverflow.com/a/25286268/540162
+def munge_path(path: str) -> str:
+	return re.sub(r'[\\:*?"|<>]+', '-', path.encode('unicode_escape').decode('utf-8'))
+
+def main():
+	if len(sys.argv) != 3:
+		print(f'Usage: {os.path.basename(__file__)} PATHS-FILE DESTINATION-DIR', file=sys.stderr)
+		sys.exit(1)
+
+	paths_file: str = sys.argv[1]
+	destination_dir: str = sys.argv[2]
+
+	if not os.path.exists(paths_file):
+		print(f'PATHS-FILE \'{paths_file}\' does not exist or is not readable', file=sys.stderr)
+		sys.exit(2)
+
+
+	print('[DEBUG] Gathering destination files list')
+	destination_files: Set[Path] = set()
+	if os.path.isdir(destination_dir):
+		for dirpath, _, files in os.walk(destination_dir):
+			for file in files:
+				path = Path(os.path.join(dirpath, file)).absolute()
+				if path.suffix == '.mp3':
+					if path in destination_files:
+						print(f'Ignoring duplicate input entry: {file}')
+					else:
+						destination_files.add(path)
+	print(f'[DEBUG] Gathered {len(destination_files)} destination files')
+
+	print('[DEBUG] Gathering source files list')
+	source_files: Set[Path] = set()
+	for line in open(paths_file, 'r'):
+		trimmed_line = line.strip()
+		if not trimmed_line:
+			continue
+
+		path = Path(trimmed_line).absolute()
+		if path.is_file():
+			if path.suffix in KNOWN_EXTENSIONS:
+				source_files.add(path)
+			else:
+				print(f'[TRACE] Unknown file extension of entry: {trimmed_line}', file=sys.stderr)
+		elif path.is_dir():
+			for dirpath, _, files in os.walk(path.absolute()):
+				# TODO: DRY (see above)
+				for file in files:
+					file_path = Path(os.path.join(dirpath, file)).absolute()
+					if file_path.suffix in KNOWN_EXTENSIONS:
+						source_files.add(file_path)
+					else:
+						print(f'[TRACE] Unknown file extension of entry: {file}', file=sys.stderr)
+		elif not path.exists():
+			print(f'[ERROR] Could not find source path: {path}')
+			sys.exit(3)
+
+	if len(source_files) == 0:
+		print('No source files to transfer')
+		sys.exit()
+	else:
+		print(f'[DEBUG] Gathered {len(source_files)} source files')
+
+	print('[DEBUG] Finding longest common prefix of source files')
+	longest_prefix = next(iter(source_files)).parent
+	for file in source_files:
+		if longest_prefix.as_posix() == '/':
+			break
+
+		while not file.as_posix().startswith(longest_prefix.as_posix()):
+			longest_prefix = longest_prefix.parent
+
+	print('[DEBUG] Filtering files already present in destination')
+	# NOTE: this assumes all filenames are unique, which should already be the case in my music library
+	#       verify via `find . -type f -not -name '*.jpg' -not -name '*.png' -not -name '*.txt' -exec basename {} \; | sort | uniq -d`
+	# TODO: add a hash check so that we can overwrite songs on the target if source
+	#       has a version which is different (new metadata, or better version)
+	for source_path in list(source_files)[:]:
+		found_destination_path: Union[Path, None] = next((destination_file for destination_file in destination_files if destination_file.stem == munge_path(source_path.stem)), None)
+		if found_destination_path:
+			source_files.remove(source_path)
+			destination_files.remove(found_destination_path)
+
+	if len(source_files) == 0:
+		print('All source files already exist in target')
+		sys.exit()
+
+	# TODO: improve prompt (print 10+ file paths or exit to `PAGER` to view before prompt so user has more info before confirming)
+	if len(destination_files) > 0:
+		delete_selection_made = False
+		while not delete_selection_made:
+			print(f'Files in destination not found in source files list:\n{destination_files}')
+			delete_input = input(f'Delete {len(destination_files)} files not found in source files list? [y/n] ')
+			if delete_input[0] == 'y' or delete_input[0] == 'Y':
+				delete_selection_made = True
+				for destination_file in destination_files:
+					destination_file.unlink()
+			elif delete_input[0] == 'n' or delete_input[0] == 'N':
+				delete_selection_made = True
+
+	# TODO: list both number of files in songs list & number of files to be transferred
+	print(f'Copying {len(source_files)} songs to {destination_dir}')
+	print(f'[TRACE] Copying the following files to destination:\n{source_files}')
+
+	for source_file in source_files:
+		source_copy_path = source_file.with_suffix('').as_posix().replace(longest_prefix.as_posix(), '', 1)
+		destination_filename = munge_path(source_copy_path)
+		destination_file_path = f'{destination_dir}/{destination_filename}.mp3'
+
+		# TODO: double-check on this
+		print(f'destination_file_path: {destination_file_path}')
+		Path(destination_file_path).parent.mkdir(parents = True, exist_ok = True)
+
+		if source_file.stem == '.mp3':
+			shutil.copy2(source_file.as_posix(), destination_file_path)
+		else:
+			subprocess.call([
+				'ffmpeg',
+				'-loglevel', 'quiet',
+				'-i', source_file.as_posix(),
+				'-codec:a', 'libmp3lame',
+				# convert to mp3 with quality level 3 vbr (average 175 kbit/s, ranges from 150-195 kbit/s)
+				# see: https://trac.ffmpeg.org/wiki/Encode/MP3#VBREncoding
+				'-q:a', '3',
+				destination_file_path
+			])
+
+
+	# TODO: improve this output (number of deleted files, number of new files, correct plural of 'files')
+	print(f'Finished copying {len(source_files)} new files to {destination_dir}')
+
+if __name__ == '__main__':
+	main()

src/.bin/music_transfer_test.py

@@ -0,0 +1,51 @@
+#!/bin/python3
+
+###
+### Tests for music transfer script functions and behaviors.
+###
+
+from music_transfer import (
+	munge_path,
+)
+import unittest
+
+class PathMungingTestCase(unittest.TestCase):
+	def test_colon_munging(self):
+		self.assertEqual('/Music/Haywyre/Haywyre - Panorama- Form',
+			munge_path('/Music/Haywyre/Haywyre - Panorama: Form'))
+
+	def test_asterisk_munging(self):
+		self.assertEqual('/Music/Other, Various Artists/P-Light - complexity.flac',
+			munge_path('/Music/Other, Various Artists/P*Light - complexity.flac'))
+
+	def test_question_mark_munging(self):
+		self.assertEqual('/Music/Other, Various Artists/They Might Be Giants - Am I Awake-.mp3',
+			munge_path('/Music/Other, Various Artists/They Might Be Giants - Am I Awake?.mp3'))
+
+	def test_quote_munging(self):
+		self.assertEqual('/media/storage0/Documents/My Music/NoteBlock/NoteBlock - -C-R-O-W-N-E-D- Kirby\'s Return to Dreamland Deluxe Remix.opus',
+			munge_path('/media/storage0/Documents/My Music/NoteBlock/NoteBlock - "C-R-O-W-N-E-D" Kirby\'s Return to Dreamland Deluxe Remix.opus'))
+
+	def test_emoji_munging(self):
+		self.assertEqual('/Music/TORLEY/TORLEY -U0001f349 - Odds & Ends/TORLEY -U0001f349 - Odds & Ends - 30 [PIANO] The Games We Played.flac',
+			munge_path('/Music/TORLEY/TORLEY 🍉 - Odds & Ends/TORLEY 🍉 - Odds & Ends - 30 [PIANO] The Games We Played.flac'))
+
+	def test_multiple_munging(self):
+		# colon and unicode (U+2019, which resembles backtick)
+		self.assertEqual('/Music/Other, Various Artists/AD-Drum-u2019n Bass/AD-Drum-u2019n Bass 01 - Jerico - Industrial Nation.flac',
+			munge_path('/Music/Other, Various Artists/AD:Drum’n Bass/AD:Drum’n Bass 01 - Jerico - Industrial Nation.flac'))
+
+		# quote and unicode (Japanese characters)
+		self.assertEqual('/Music/Camellia/-u304b-u3081-u308a-u3042 - Camellia -Guest Tracks- Summary & VIPs 01/-u304b-u3081-u308a-u3042 - Camellia -Guest Tracks- Summary & VIPs 01 - 14 Feelin Sky (Camellia\'s -200step- Self-remix).flac',
+			munge_path('/Music/Camellia/かめりあ - Camellia "Guest Tracks" Summary & VIPs 01/かめりあ - Camellia "Guest Tracks" Summary & VIPs 01 - 14 Feelin Sky (Camellia\'s ""200step"" Self-remix).flac'))
+
+		# unicode (U+B0, degree symbol)
+		self.assertEqual('/Music/Caravan Palace/Caravan Palace - I-xb0_-xb0I',
+			munge_path('/Music/Caravan Palace/Caravan Palace - I°_°I'))
+
+		# unicode (U+2606, outlined star symbol)
+		self.assertEqual('/Music/Other, Various Artists/Various Artists - SONIC-u2606FRONTLINE v1.0/SONIC-u2606FRONTLINE v1.0 06 - Tanchiky - STEP BY STEP.flac',
+			munge_path('/Music/Other, Various Artists/Various Artists - SONIC☆FRONTLINE v1.0/SONIC☆FRONTLINE v1.0 06 - Tanchiky - STEP BY STEP.flac'))
+
+if __name__ == '__main__':
+	unittest.main()

src/.bin/open-runelite.sh

@@ -23,7 +23,7 @@ MAX_AGE_BEFORE_REBUILD="$(( 18 * 60 * 60 ))" # 18 hours
 UPSTREAM_REMOTE='upstream'
 UPSTREAM_BRANCH='master'
 for branch in "${BRANCHES_TO_MERGE[@]}"; do
-	if grep -qE '^internal' <<< "$branch"; then
+	if grep -q 'internal' <<< "$branch"; then
 		UPSTREAM_REMOTE='internal-upstream'
 		UPSTREAM_BRANCH='internal'
 		break
@@ -50,8 +50,8 @@ DIR="$(dirname "${BASH_SOURCE[0]}")"
 RUNELITE_TARGET_COMMIT_FILENAME="$DIR/commit"
 function build_client {
 	# Update local dev branch and build
-	git checkout dev || git checkout -b dev upstream/master
-	git reset --hard upstream/master
+	git checkout dev || git checkout -b dev
+	git reset --hard "$UPSTREAM_REMOTE/$UPSTREAM_BRANCH"
 	git merge --no-gpg-sign --no-edit "${BRANCHES_TO_MERGE[@]}" || exit 1
 	mvn clean package -DskipTests -U || exit 1
 	git checkout -
@@ -102,4 +102,4 @@ else
 fi
 
 # run the dev client
-nohup java -ea -Drunelite.pluginhub.version="${PREVIOUS_TAG}" -jar "$CACHED_JAR" --developer-mode --debug </dev/null >/dev/null 2>&1 &
+nohup java -ea -Drunelite.pluginhub.version="${PREVIOUS_TAG}" -jar "$CACHED_JAR" --developer-mode --debug --profile=main </dev/null >/dev/null 2>&1 &

src/.bin/runelite-blog.sh

@@ -111,7 +111,7 @@ git_username_trimmed="$(git config user.name | cut -d ' ' -f '1')"
 default_date='+1 day'
 BLOG_DATE="${blog_date:-$default_date}"
 BLOG_AUTHOR="${blog_author:-$git_username_trimmed}"
-git_shortlog_options=(--no-merges --perl-regexp '--author=^((?!Runelite auto updater|RuneLite Cache-Code Autoupdater).*)$' "${latest_tag}"..upstream/master)
+git_shortlog_options=(--no-merges --perl-regexp '--author=^((?!Runelite auto updater|RuneLite Cache-Code Autoupdater|RuneLite updater).*)$' "${latest_tag}"..upstream/master)
 SHORTLOG_SINCE_LAST_TAG="$(git -C "$repo_path" --no-pager shortlog "${git_shortlog_options[@]}")"
 NUMBER_OF_CONTRIBUTORS="$(git -C "$repo_path" --no-pager shortlog -s "${git_shortlog_options[@]}" | wc -l)"
 BLOG_POST_DATE="$(date -d "${BLOG_DATE} 10:00am" +'%Y-%m-%d-%H-%M')"
@@ -173,3 +173,4 @@ ${SHORTLOG_SINCE_LAST_TAG}
 EOF
 
 echo 'Blog branch and structure created!'
+git status

src/.gitconfig

@@ -34,6 +34,7 @@
 	pullrsurp = !git pull --rebase && git submodule update --recursive && git push
 	reauthor = "!git commit --amend --no-edit --author=\"$(git config user.name) <$(git config user.email)>\""
 	reword = commit --amend
+	r- = rebase -
 	shortstat = diff --shortstat
 	stat = diff --stat
 	sur = submodule update --recursive
@@ -73,9 +74,7 @@
 [log]
 	follow = true
 [merge]
-	# TODO: Switch this to the even better "zdiff3" (zealous diff3) setting once
-	# git 2.35 is available in Debian stable
-	conflictstyle = diff3
+	conflictstyle = zdiff3
 [notes]
 	displayRef = refs/notes/*
 [pager]