version 2.1

Added checks for existing Ticket and entry in Title Database

.gitignore

@@ -18,3 +18,6 @@ venv/
 =======
 
 *.pyc
+/build/
+/dist/
+/custom-install-finalize.3dsx

ci-gui.py

@@ -6,7 +6,7 @@
 
 from os import environ, scandir
 from os.path import abspath, basename, dirname, join, isfile
-from sys import exc_info, platform
+import sys
 from threading import Thread, Lock
 from time import strftime
 from traceback import format_exception
@@ -29,9 +29,15 @@ if TYPE_CHECKING:
     from os import PathLike
     from typing import Dict, List, Union
 
-is_windows = platform == 'win32'
+frozen = getattr(sys, 'frozen', None)
+is_windows = sys.platform == 'win32'
 taskbar = None
 if is_windows:
+    if frozen:
+        # attempt to fix loading tcl/tk when running from a path with non-latin characters
+        tkinter_path = dirname(tk.__file__)
+        tcl_path = join(tkinter_path, 'tcl8.6')
+        environ['TCL_LIBRARY'] = 'lib/tkinter/tcl8.6'
     try:
         import comtypes.client as cc
 
@@ -39,7 +45,7 @@ if is_windows:
 
         taskbar = cc.CreateObject('{56FDF344-FD6D-11D0-958A-006097C9A090}', interface=tbl.ITaskbarList3)
         taskbar.HrInit()
-    except ModuleNotFoundError:
+    except (ModuleNotFoundError, UnicodeEncodeError, AttributeError):
         pass
 
 file_parent = dirname(abspath(__file__))
@@ -190,7 +196,8 @@ class TitleReadFailResults(tk.Toplevel):
 
 
 class InstallResults(tk.Toplevel):
-    def __init__(self, parent: tk.Tk = None, *, install_state: 'Dict[str, List[str]]', copied_3dsx: bool):
+    def __init__(self, parent: tk.Tk = None, *, install_state: 'Dict[str, List[str]]', copied_3dsx: bool,
+                 application_count: int):
         super().__init__(parent)
         self.parent = parent
 
@@ -223,6 +230,11 @@ class InstallResults(tk.Toplevel):
         if install_state['installed'] and copied_3dsx:
             message += '\n\ncustom-install-finalize has been copied to the SD card.'
 
+        if application_count >= 300:
+            message += (f'\n\nWarning: {application_count} installed applications were detected.\n'
+                        f'The HOME Menu will only show 300 icons.\n'
+                        f'Some applications (not updates or DLC) will need to be deleted.')
+
         message_label = ttk.Label(outer_container, text=message)
         message_label.grid(row=0, column=0, sticky=tk.NSEW, padx=10, pady=10)
 
@@ -300,12 +312,14 @@ class CustomInstallGUI(ttk.Frame):
                 sd_selected.delete('1.0', tk.END)
                 sd_selected.insert(tk.END, f)
 
-                sd_msed_path = find_first_file([join(f, 'gm9', 'out', 'movable.sed'), join(f, 'movable.sed')])
-                if sd_msed_path:
-                    self.log('Found movable.sed on SD card at ' + sd_msed_path)
-                    box = self.file_picker_textboxes['movable.sed']
-                    box.delete('1.0', tk.END)
-                    box.insert(tk.END, sd_msed_path)
+                for filename in ['boot9.bin', 'seeddb.bin', 'movable.sed']:
+                    path = auto_input_filename(self, f, filename)
+                    if filename == 'boot9.bin':
+                        self.check_b9_loaded()
+                        self.enable_buttons()
+                    if filename == 'seeddb.bin':
+                        load_seeddb(path)
+
 
         sd_type_label = ttk.Label(file_pickers, text='SD root')
         sd_type_label.grid(row=0, column=0)
@@ -318,6 +332,16 @@ class CustomInstallGUI(ttk.Frame):
 
         self.file_picker_textboxes['sd'] = sd_selected
 
+        def auto_input_filename(self, f, filename):
+            sd_msed_path = find_first_file([join(f, 'gm9', 'out', filename), join(f, filename)])
+            if sd_msed_path:
+                self.log('Found ' + filename + ' on SD card at ' + sd_msed_path)
+                if filename.endswith('bin'):
+                    filename = filename.split('.')[0]
+                box = self.file_picker_textboxes[filename]
+                box.delete('1.0', tk.END)
+                box.insert(tk.END, sd_msed_path)
+                return sd_msed_path
         # This feels so wrong.
         def create_required_file_picker(type_name, types, default, row, callback=lambda filename: None):
             def internal_callback():
@@ -474,7 +498,7 @@ class CustomInstallGUI(ttk.Frame):
         self.log(f'custom-install {CI_VERSION} - https://github.com/ihaveamac/custom-install', status=False)
 
         if is_windows and not taskbar:
-            self.log('Note: comtypes module not found.')
+            self.log('Note: Could not load taskbar lib.')
             self.log('Note: Progress will not be shown in the Windows taskbar.')
 
         self.log('Ready.')
@@ -618,7 +642,6 @@ class CustomInstallGUI(ttk.Frame):
         for path in self.readers.keys():
             self.update_status(path, InstallStatus.Waiting)
         self.disable_buttons()
-        self.log('Starting install...')
 
         if taskbar:
             taskbar.SetProgressState(self.hwnd, tbl.TBPF_NORMAL)
@@ -628,6 +651,16 @@ class CustomInstallGUI(ttk.Frame):
                                   skip_contents=self.skip_contents_var.get() == 1,
                                   overwrite_saves=self.overwrite_saves_var.get() == 1)
 
+        if not installer.check_for_id0():
+            self.show_error(f'id0 {installer.crypto.id0.hex()} was not found inside "Nintendo 3DS" on the SD card.\n'
+                            f'\n'
+                            f'Before using custom-install, you should use this SD card on the appropriate console.\n'
+                            f'\n'
+                            f'Otherwise, make sure the correct movable.sed is being used.')
+            return
+
+        self.log('Starting install...')
+
         # use the treeview which has been sorted alphabetically
         readers_final = []
         for k in self.treeview.get_children():
@@ -681,16 +714,19 @@ class CustomInstallGUI(ttk.Frame):
 
         def install():
             try:
-                result, copied_3dsx = installer.start()
+                result, copied_3dsx, application_count = installer.start()
                 if result:
-                    result_window = InstallResults(self.parent, install_state=result, copied_3dsx=copied_3dsx)
+                    result_window = InstallResults(self.parent,
+                                                   install_state=result,
+                                                   copied_3dsx=copied_3dsx,
+                                                   application_count=application_count)
                     result_window.focus()
                 elif result is None:
                     self.show_error("An error occurred when trying to run save3ds_fuse.\n"
                                     "Either title.db doesn't exist, or save3ds_fuse couldn't be run.")
                     self.open_console()
             except:
-                installer.event.on_error(exc_info())
+                installer.event.on_error(sys.exc_info())
             finally:
                 self.enable_buttons()
 

custominstall.py

@@ -6,11 +6,12 @@
 
 from argparse import ArgumentParser
 from enum import Enum
+from glob import glob
+import gzip
 from os import makedirs, rename, scandir
 from os.path import dirname, join, isdir, isfile
 from random import randint
 from hashlib import sha256
-from locale import getpreferredencoding
 from pprint import pformat
 from shutil import copyfile, copy2, rmtree
 import sys
@@ -43,7 +44,7 @@ if is_windows:
 else:
     from os import statvfs
 
-CI_VERSION = '2.1b2'
+CI_VERSION = '2.1'
 
 # used to run the save3ds_fuse binary next to the script
 frozen = getattr(sys, 'frozen', False)
@@ -270,6 +271,10 @@ class CustomInstall:
         free_space = get_free_space(self.sd)
         return total_size, free_space
 
+    def check_for_id0(self):
+        sd_path = join(self.sd, 'Nintendo 3DS', self.crypto.id0.hex())
+        return isdir(sd_path)
+
     def start(self):
         if frozen:
             save3ds_fuse_path = join(script_dir, 'bin', 'save3ds_fuse')
@@ -279,7 +284,7 @@ class CustomInstall:
             save3ds_fuse_path += '.exe'
         if not isfile(save3ds_fuse_path):
             self.log("Couldn't find " + save3ds_fuse_path, 2)
-            return None, False
+            return None, False, 0
 
         crypto = self.crypto
         # TODO: Move a lot of these into their own methods
@@ -290,6 +295,8 @@ class CustomInstall:
                               f'please remove extra directories')
         elif len(id1s) == 0:
             raise SDPathError(f'Could not find a suitable id1 directory for id0 {crypto.id0.hex()}')
+        id1 = id1s[0]
+        sd_path = join(sd_path, id1)
 
         if self.cifinish_out:
             cifinish_path = self.cifinish_out
@@ -304,7 +311,41 @@ class CustomInstall:
                      f'This could mean an issue with the SD card or the filesystem. Please check it for errors.\n'
                      f'It is also possible, though less likely, to be an issue with custom-install.\n'
                      f'Exiting now to prevent possible issues. If you want to try again, delete cifinish.bin from the SD card and re-run custom-install.')
-            return None, False
+            return None, False, 0
+
+        db_path = join(sd_path, 'dbs')
+        titledb_path = join(db_path, 'title.db')
+        importdb_path = join(db_path, 'import.db')
+        if not isfile(titledb_path):
+            makedirs(db_path, exist_ok=True)
+            with gzip.open(join(script_dir, 'title.db.gz')) as f:
+                tdb = f.read()
+
+            self.log(f'Creating title.db...')
+            with open(titledb_path, 'wb') as o:
+                with self.crypto.create_ctr_io(Keyslot.SD, o, self.crypto.sd_path_to_iv('/dbs/title.db')) as e:
+                    e.write(tdb)
+
+                    cmac = crypto.create_cmac_object(Keyslot.CMACSDNAND)
+                    cmac_data = [b'CTR-9DB0', 0x2.to_bytes(4, 'little'), tdb[0x100:0x200]]
+                    cmac.update(sha256(b''.join(cmac_data)).digest())
+
+                    e.seek(0)
+                    e.write(cmac.digest())
+
+            self.log(f'Creating import.db...')
+            with open(importdb_path, 'wb') as o:
+                with self.crypto.create_ctr_io(Keyslot.SD, o, self.crypto.sd_path_to_iv('/dbs/import.db')) as e:
+                    e.write(tdb)
+
+                    cmac = crypto.create_cmac_object(Keyslot.CMACSDNAND)
+                    cmac_data = [b'CTR-9DB0', 0x3.to_bytes(4, 'little'), tdb[0x100:0x200]]
+                    cmac.update(sha256(b''.join(cmac_data)).digest())
+
+                    e.seek(0)
+                    e.write(cmac.digest())
+
+            del tdb
 
         with TemporaryDirectory(suffix='-custom-install') as tempdir:
             # set up the common arguments for the two times we call save3ds_fuse
@@ -327,7 +368,7 @@ class CustomInstall:
             out = subprocess.run(save3ds_fuse_common_args + ['-x'],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT,
-                                 encoding=getpreferredencoding(),
+                                 encoding='utf-8',
                                  **extra_kwargs)
             if out.returncode:
                 for l in out.stdout.split('\n'):
@@ -335,9 +376,7 @@ class CustomInstall:
                 self.log('Command line:')
                 for l in pformat(out.args).split('\n'):
                     self.log(l)
-                return None, False
-
-            sd_path = join(sd_path, id1s[0])
+                return None, False, 0
 
             if self.seeddb:
                 load_seeddb(self.seeddb)
@@ -570,7 +609,7 @@ class CustomInstall:
                 out = subprocess.run(save3ds_fuse_common_args + ['-i'],
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.STDOUT,
-                                     encoding=getpreferredencoding(),
+                                     encoding='utf-8',
                                      **extra_kwargs)
                 if out.returncode:
                     for l in out.stdout.split('\n'):
@@ -585,7 +624,13 @@ class CustomInstall:
                     self.event.update_status(path, InstallStatus.Done)
 
             copied = False
+            # launchable applications, not DLC or update data
+            application_count = len(glob(join(tempdir, '00040000*')))
             if install_state['installed']:
+                if application_count >= 300:
+                    self.log(f'{application_count} installed applications were detected.', 1)
+                    self.log('The HOME Menu will only show 300 icons.', 1)
+                    self.log('Some applications (not updates or DLC) will need to be deleted.', 1)
                 finalize_3dsx_orig_path = join(script_dir, 'custom-install-finalize.3dsx')
                 hb_dir = join(self.sd, '3ds')
                 finalize_3dsx_path = join(hb_dir, 'custom-install-finalize.3dsx')
@@ -601,7 +646,7 @@ class CustomInstall:
                 if copied:
                     self.log('custom-install-finalize has been copied to the SD card.')
 
-            return install_state, copied
+            return install_state, copied, application_count
 
     def get_sd_path(self):
         sd_path = join(self.sd, 'Nintendo 3DS', self.crypto.id0.hex())
@@ -682,6 +727,10 @@ if __name__ == "__main__":
     installer.event.update_percentage += percent_handle
     installer.event.on_error += error
 
+    if not installer.check_for_id0():
+        installer.event.on_error(f'Could not find id0 directory {installer.crypto.id0.hex()} '
+                                 f'inside Nintendo 3DS directory.')
+
     installer.prepare_titles(args.cia)
 
     if not args.skip_contents:
@@ -692,7 +741,11 @@ if __name__ == "__main__":
                                        f'Free space: {free_space / (1024 * 1024):0.2f} MiB')
             sys.exit(1)
 
-    result, copied_3dsx = installer.start()
+    result, copied_3dsx, application_count = installer.start()
     if result is False:
         # save3ds_fuse failed
         installer.log('NOTE: Once save3ds_fuse is fixed, run the same command again with --skip-contents')
+    if application_count >= 300:
+        installer.log(f'\n\nWarning: {application_count} installed applications were detected.\n'
+                      f'The HOME Menu will only show 300 icons.\n'
+                      f'Some applications (not updates or DLC) will need to be deleted.')

finalize/source/main.c

@@ -197,31 +197,99 @@ fail:
 	return -1;
 }
 
+Result check_title_exist(u64 title_id, u64 *ticket_ids, u32 ticket_ids_length,  u64 *title_ids, u32 title_ids_length)
+{
+	Result ret = -2;
+
+	for (u32 i = 0; i < ticket_ids_length; i++)
+	{
+		if (ticket_ids[i] == title_id)
+		{
+			ret++;
+			break;
+		}
+	}
+
+	for (u32 i = 0; i < title_ids_length; i++)
+	{
+		if (title_ids[i] == title_id)
+		{
+			ret++;
+			break;
+		}
+	}
+
+	return ret;
+}
+
 void finalize_install(void)
 {
 	Result res;
 	Handle ticketHandle;
 	struct ticket_dumb ticket_buf;
-	struct finish_db_entry_final *entries;
-	int title_count;
+	struct finish_db_entry_final *entries = NULL;
+	int title_count;	
 
-	title_count = load_cifinish(CIFINISH_PATH, &entries);
-	if (title_count == -1)
+	u32 titles_read;
+	u32 tickets_read;
+
+	res = AM_GetTitleCount(MEDIATYPE_SD, &titles_read);
+
+	if (R_FAILED(res))
 	{
-		free(entries);
 		return;
 	}
-	if (title_count == 0)
+
+	res = AM_GetTicketCount(&tickets_read);
+
+	if (R_FAILED(res))
+	{
+		return;
+	}
+
+	u64 *installed_ticket_ids = malloc(sizeof(u64) * tickets_read );
+	u64 *installed_title_ids  = malloc(sizeof(u64) * titles_read  );
+
+	res = AM_GetTitleList(&titles_read, MEDIATYPE_SD, titles_read, installed_title_ids);
+
+	if (R_FAILED(res))
+	{
+		goto exit;
+	}
+
+	res = AM_GetTicketList(&tickets_read, tickets_read, 0, installed_ticket_ids);
+
+	if (R_FAILED(res))
+	{
+		goto exit;
+	}
+
+	title_count = load_cifinish(CIFINISH_PATH, &entries);
+
+	if (title_count == -1)
+	{
+		goto exit;
+	}
+	else if (title_count == 0)
 	{
 		printf("No titles to finalize.\n");
-		free(entries);
-		return;
+		goto exit;
 	}
 
 	memcpy(&ticket_buf, basetik_bin, basetik_bin_size);
 
+	Result exist_res = 0;
+
 	for (int i = 0; i < title_count; ++i)
 	{
+		exist_res = check_title_exist(entries[i].title_id, installed_ticket_ids, tickets_read, installed_title_ids, titles_read);
+
+		if (R_SUCCEEDED(exist_res))
+		{
+			printf("No need to finalize %016llx, skipping...\n", entries[i].title_id);
+			continue;
+		}
+
 		printf("Finalizing %016llx...\n", entries[i].title_id);
 
 		ticket_buf.title_id_be = __builtin_bswap64(entries[i].title_id);
@@ -231,8 +299,7 @@ void finalize_install(void)
 		{
 			printf("Failed to begin ticket install: %08lx\n", res);
 			AM_InstallTicketAbort(ticketHandle);
-			free(entries);
-			return;
+			goto exit;
 		}
 
 		res = FSFILE_Write(ticketHandle, NULL, 0, &ticket_buf, sizeof(struct ticket_dumb), 0);
@@ -240,8 +307,7 @@ void finalize_install(void)
 		{
 			printf("Failed to write ticket: %08lx\n", res);
 			AM_InstallTicketAbort(ticketHandle);
-			free(entries);
-			return;
+			goto exit;
 		}
 
 		res = AM_InstallTicketFinish(ticketHandle);
@@ -249,8 +315,7 @@ void finalize_install(void)
 		{
 			printf("Failed to finish ticket install: %08lx\n", res);
 			AM_InstallTicketAbort(ticketHandle);
-			free(entries);
-			return;
+			goto exit;
 		}
 
 		if (entries[i].has_seed)
@@ -267,7 +332,12 @@ void finalize_install(void)
 	printf("Deleting %s...\n", CIFINISH_PATH);
 	unlink(CIFINISH_PATH);
 
+	exit:
+
 	free(entries);
+	free(installed_ticket_ids);
+	free(installed_title_ids);
+	return;
 }
 
 int main(int argc, char* argv[])
@@ -276,7 +346,7 @@ int main(int argc, char* argv[])
 	gfxInitDefault();
 	consoleInit(GFX_TOP, NULL);
 
-	printf("custom-install-finalize v1.5\n");
+	printf("custom-install-finalize v1.6\n");
 
 	finalize_install();
 	// print this at the end in case it gets pushed off the screen

make-standalone.bat

@@ -1,11 +1,13 @@
 mkdir build
 mkdir dist
-cxfreeze ci-gui.py --target-dir=build\custom-install-standalone --base-name=Win32GUI
+python setup-cxfreeze.py build_exe --build-exe=build\custom-install-standalone
 mkdir build\custom-install-standalone\bin
 copy TaskbarLib.tlb build\custom-install-standalone
 copy bin\win32\save3ds_fuse.exe build\custom-install-standalone\bin
 copy bin\README build\custom-install-standalone\bin
 copy custom-install-finalize.3dsx build\custom-install-standalone
+copy title.db.gz build\custom-install-standalone
 copy extras\windows-quickstart.txt build\custom-install-standalone
+copy extras\run_with_cmd.bat build\custom-install-standalone
 copy LICENSE.md build\custom-install-standalone
 python -m zipfile -c dist\custom-install-standalone.zip build\custom-install-standalone

requirements-win32.txt

@@ -1,2 +1,2 @@
 -r requirements.txt
-comtypes==1.1.7
+comtypes==1.1.10

requirements.txt

@@ -1,2 +1,2 @@
 events==0.4
-pyctr==0.4.5
+pyctr>=0.4,<0.6

setup-cxfreeze.py

@@ -0,0 +1,19 @@
+import sys
+from cx_Freeze import setup, Executable
+
+if sys.platform == 'win32':
+    executables = [
+        Executable('ci-gui.py', target_name='ci-gui-console'),
+        Executable('ci-gui.py', target_name='ci-gui', base='Win32GUI'),
+    ]
+else:
+    executables = [
+        Executable('ci-gui.py', target_name='ci-gui'),
+    ]
+
+setup(
+    name = "ci-gui",
+    version = "2.1b4",
+    description = "Installs a title directly to an SD card for the Nintendo 3DS",
+    executables = executables
+)