version 2.1b3

ci-gui.py

@@ -16,16 +16,18 @@ import tkinter.filedialog as fd
 import tkinter.messagebox as mb
 from typing import TYPE_CHECKING
 
+from pyctr.crypto import MissingSeedError, CryptoEngine, load_seeddb
 from pyctr.crypto.engine import b9_paths
 from pyctr.util import config_dirs
 from pyctr.type.cdn import CDNError
 from pyctr.type.cia import CIAError
 from pyctr.type.tmd import TitleMetadataError
 
-from custominstall import CustomInstall, CI_VERSION, load_cifinish, InvalidCIFinishError
+from custominstall import CustomInstall, CI_VERSION, load_cifinish, InvalidCIFinishError, InstallStatus
 
 if TYPE_CHECKING:
-    from typing import Dict, List
+    from os import PathLike
+    from typing import Dict, List, Union
 
 is_windows = platform == 'win32'
 taskbar = None
@@ -70,6 +72,18 @@ default_b9_path = find_first_file(b9_paths)
 default_seeddb_path = find_first_file(seeddb_paths)
 default_movable_sed_path = find_first_file([join(file_parent, 'movable.sed')])
 
+if default_seeddb_path:
+    load_seeddb(default_seeddb_path)
+
+statuses = {
+    InstallStatus.Waiting: 'Waiting',
+    InstallStatus.Starting: 'Starting',
+    InstallStatus.Writing: 'Writing',
+    InstallStatus.Finishing: 'Finishing',
+    InstallStatus.Done: 'Done',
+    InstallStatus.Failed: 'Failed',
+}
+
 
 class ConsoleFrame(ttk.Frame):
     def __init__(self, parent: tk.BaseWidget = None, starting_lines: 'List[str]' = None):
@@ -176,7 +190,8 @@ class TitleReadFailResults(tk.Toplevel):
 
 
 class InstallResults(tk.Toplevel):
-    def __init__(self, parent: tk.Tk = None, *, install_state: 'Dict[str, List[str]]', copied_3dsx: bool):
+    def __init__(self, parent: tk.Tk = None, *, install_state: 'Dict[str, List[str]]', copied_3dsx: bool,
+                 application_count: int):
         super().__init__(parent)
         self.parent = parent
 
@@ -209,6 +224,11 @@ class InstallResults(tk.Toplevel):
         if install_state['installed'] and copied_3dsx:
             message += '\n\ncustom-install-finalize has been copied to the SD card.'
 
+        if application_count >= 300:
+            message += (f'\n\nWarning: {application_count} installed applications were detected.\n'
+                        f'The HOME Menu will only show 300 icons.\n'
+                        f'Some applications (not updates or DLC) will need to be deleted.')
+
         message_label = ttk.Label(outer_container, text=message)
         message_label.grid(row=0, column=0, sticky=tk.NSEW, padx=10, pady=10)
 
@@ -235,6 +255,7 @@ class InstallResults(tk.Toplevel):
 
 class CustomInstallGUI(ttk.Frame):
     console = None
+    b9_loaded = False
 
     def __init__(self, parent: tk.Tk = None):
         super().__init__(parent)
@@ -304,13 +325,14 @@ class CustomInstallGUI(ttk.Frame):
         self.file_picker_textboxes['sd'] = sd_selected
 
         # This feels so wrong.
-        def create_required_file_picker(type_name, types, default, row):
+        def create_required_file_picker(type_name, types, default, row, callback=lambda filename: None):
             def internal_callback():
                 f = fd.askopenfilename(parent=parent, title='Select ' + type_name, filetypes=types,
                                        initialdir=file_parent)
                 if f:
                     selected.delete('1.0', tk.END)
                     selected.insert(tk.END, f)
+                    callback(f)
 
             type_label = ttk.Label(file_pickers, text=type_name)
             type_label.grid(row=row, column=0)
@@ -325,8 +347,15 @@ class CustomInstallGUI(ttk.Frame):
 
             self.file_picker_textboxes[type_name] = selected
 
-        create_required_file_picker('boot9', [('boot9 file', '*.bin')], default_b9_path, 1)
-        create_required_file_picker('seeddb', [('seeddb file', '*.bin')], default_seeddb_path, 2)
+        def b9_callback(path: 'Union[PathLike, bytes, str]'):
+            self.check_b9_loaded()
+            self.enable_buttons()
+
+        def seeddb_callback(path: 'Union[PathLike, bytes, str]'):
+            load_seeddb(path)
+
+        create_required_file_picker('boot9', [('boot9 file', '*.bin')], default_b9_path, 1, b9_callback)
+        create_required_file_picker('seeddb', [('seeddb file', '*.bin')], default_seeddb_path, 2, seeddb_callback)
         create_required_file_picker('movable.sed', [('movable.sed file', '*.sed')], default_movable_sed_path, 3)
 
         # ---------------------------------------------------------------- #
@@ -404,14 +433,16 @@ class CustomInstallGUI(ttk.Frame):
 
         self.treeview = ttk.Treeview(treeview_frame, yscrollcommand=treeview_scrollbar.set)
         self.treeview.grid(row=0, column=0, sticky=tk.NSEW)
-        self.treeview.configure(columns=('filepath', 'titleid', 'titlename'), show='headings')
+        self.treeview.configure(columns=('filepath', 'titleid', 'titlename', 'status'), show='headings')
 
         self.treeview.column('filepath', width=200, anchor=tk.W)
         self.treeview.heading('filepath', text='File path')
-        self.treeview.column('titleid', width=50, anchor=tk.W)
+        self.treeview.column('titleid', width=70, anchor=tk.W)
         self.treeview.heading('titleid', text='Title ID')
         self.treeview.column('titlename', width=150, anchor=tk.W)
         self.treeview.heading('titlename', text='Title name')
+        self.treeview.column('status', width=20, anchor=tk.W)
+        self.treeview.heading('status', text='Status')
 
         treeview_scrollbar.configure(command=self.treeview.yview)
 
@@ -454,7 +485,13 @@ class CustomInstallGUI(ttk.Frame):
 
         self.log('Ready.')
 
-        self.disable_during_install = (add_cias, add_dirs, remove_selected, start, *self.file_picker_textboxes.values())
+        self.require_boot9 = (add_cias, add_cdn, add_dirs, remove_selected, start)
+
+        self.disable_buttons()
+        self.check_b9_loaded()
+        self.enable_buttons()
+        if not self.b9_loaded:
+            self.log('Note: boot9 was not auto-detected. Please choose it before adding any titles.')
 
     def sort_treeview(self):
         l = [(self.treeview.set(k, 'titlename'), k) for k in self.treeview.get_children()]
@@ -464,7 +501,23 @@ class CustomInstallGUI(ttk.Frame):
         for idx, pair in enumerate(l):
             self.treeview.move(pair[1], '', idx)
 
+    def check_b9_loaded(self):
+        if not self.b9_loaded:
+            boot9 = self.file_picker_textboxes['boot9'].get('1.0', tk.END).strip()
+            try:
+                tmp_crypto = CryptoEngine(boot9=boot9)
+                self.b9_loaded = tmp_crypto.b9_keys_set
+            except:
+                return False
+        return self.b9_loaded
+
+    def update_status(self, path: 'Union[PathLike, bytes, str]', status: InstallStatus):
+        self.treeview.set(path, 'status', statuses[status])
+
     def add_cia(self, path):
+        if not self.check_b9_loaded():
+            # this shouldn't happen
+            return False, 'Please choose boot9 first'
         path = abspath(path)
         if path in self.readers:
             return False, 'File already in list'
@@ -472,6 +525,8 @@ class CustomInstallGUI(ttk.Frame):
             reader = CustomInstall.get_reader(path)
         except (CIAError, CDNError, TitleMetadataError):
             return False, 'Failed to read as a CIA or CDN title, probably corrupt'
+        except MissingSeedError:
+            return False, 'Latest seeddb.bin is required, check the README for details'
         except Exception as e:
             return False, f'Exception occurred: {type(e).__name__}: {e}'
 
@@ -481,7 +536,8 @@ class CustomInstallGUI(ttk.Frame):
             title_name = reader.contents[0].exefs.icon.get_app_title().short_desc
         except:
             title_name = '(No title)'
-        self.treeview.insert('', tk.END, text=path, iid=path, values=(path, reader.tmd.title_id, title_name))
+        self.treeview.insert('', tk.END, text=path, iid=path,
+                             values=(path, reader.tmd.title_id, title_name, statuses[InstallStatus.Waiting]))
         self.readers[path] = reader
         return True, ''
 
@@ -532,25 +588,26 @@ class CustomInstallGUI(ttk.Frame):
         mb.showinfo('Info', message, parent=self.parent)
 
     def disable_buttons(self):
-        for b in self.disable_during_install:
+        for b in self.require_boot9:
+            b.config(state=tk.DISABLED)
+        for b in self.file_picker_textboxes.values():
             b.config(state=tk.DISABLED)
 
     def enable_buttons(self):
-        for b in self.disable_during_install:
+        if self.b9_loaded:
+            for b in self.require_boot9:
+                b.config(state=tk.NORMAL)
+        for b in self.file_picker_textboxes.values():
             b.config(state=tk.NORMAL)
 
     def start_install(self):
         sd_root = self.file_picker_textboxes['sd'].get('1.0', tk.END).strip()
-        boot9 = self.file_picker_textboxes['boot9'].get('1.0', tk.END).strip()
         seeddb = self.file_picker_textboxes['seeddb'].get('1.0', tk.END).strip()
         movable_sed = self.file_picker_textboxes['movable.sed'].get('1.0', tk.END).strip()
 
         if not sd_root:
             self.show_error('SD root is not specified.')
             return
-        if not boot9:
-            self.show_error('boot9 is not specified.')
-            return
         if not movable_sed:
             self.show_error('movable.sed is not specified.')
             return
@@ -560,29 +617,28 @@ class CustomInstallGUI(ttk.Frame):
                                     'Continue?'):
                 return
 
-        self.disable_buttons()
-
         if not len(self.readers):
             self.show_error('There are no titles added to install.')
             return
 
+        for path in self.readers.keys():
+            self.update_status(path, InstallStatus.Waiting)
+        self.disable_buttons()
         self.log('Starting install...')
 
         if taskbar:
             taskbar.SetProgressState(self.hwnd, tbl.TBPF_NORMAL)
 
-        installer = CustomInstall(boot9=boot9,
-                                  seeddb=seeddb,
-                                  movable=movable_sed,
+        installer = CustomInstall(movable=movable_sed,
                                   sd=sd_root,
                                   skip_contents=self.skip_contents_var.get() == 1,
                                   overwrite_saves=self.overwrite_saves_var.get() == 1)
 
         # use the treeview which has been sorted alphabetically
-        #installer.readers = self.readers.values()
         readers_final = []
         for k in self.treeview.get_children():
-            readers_final.append(self.readers[self.treeview.set(k, 'filepath')])
+            filepath = self.treeview.set(k, 'filepath')
+            readers_final.append((self.readers[filepath], filepath))
 
         installer.readers = readers_final
 
@@ -618,6 +674,7 @@ class CustomInstallGUI(ttk.Frame):
         installer.event.update_percentage += ci_update_percentage
         installer.event.on_error += ci_on_error
         installer.event.on_cia_start += ci_on_cia_start
+        installer.event.update_status += self.update_status
 
         if self.skip_contents_var.get() != 1:
             total_size, free_space = installer.check_size()
@@ -630,9 +687,12 @@ class CustomInstallGUI(ttk.Frame):
 
         def install():
             try:
-                result, copied_3dsx = installer.start()
+                result, copied_3dsx, application_count = installer.start()
                 if result:
-                    result_window = InstallResults(self.parent, install_state=result, copied_3dsx=copied_3dsx)
+                    result_window = InstallResults(self.parent,
+                                                   install_state=result,
+                                                   copied_3dsx=copied_3dsx,
+                                                   application_count=application_count)
                     result_window.focus()
                 elif result is None:
                     self.show_error("An error occurred when trying to run save3ds_fuse.\n"

custominstall.py

@@ -5,6 +5,9 @@
 # You can find the full license text in LICENSE.md in the root of this project.
 
 from argparse import ArgumentParser
+from enum import Enum
+from glob import glob
+import gzip
 from os import makedirs, rename, scandir
 from os.path import dirname, join, isdir, isfile
 from random import randint
@@ -21,7 +24,7 @@ import subprocess
 
 if TYPE_CHECKING:
     from os import PathLike
-    from typing import List, Union
+    from typing import List, Union, Tuple
 
 from events import Events
 
@@ -42,7 +45,7 @@ if is_windows:
 else:
     from os import statvfs
 
-CI_VERSION = '2.1b1'
+CI_VERSION = '2.1b3'
 
 # used to run the save3ds_fuse binary next to the script
 frozen = getattr(sys, 'frozen', False)
@@ -74,6 +77,15 @@ class InvalidCIFinishError(Exception):
     pass
 
 
+class InstallStatus(Enum):
+    Waiting = 0
+    Starting = 1
+    Writing = 2
+    Finishing = 3
+    Done = 4
+    Failed = 5
+
+
 def get_free_space(path: 'Union[PathLike, bytes, str]'):
     if is_windows:
         lpSectorsPerCluster = c_ulonglong(0)
@@ -191,15 +203,15 @@ def get_install_size(title: 'Union[CIAReader, CDNReader]'):
 
 
 class CustomInstall:
-    def __init__(self, boot9, seeddb, movable, sd, cifinish_out=None,
-                 overwrite_saves=False, skip_contents=False):
+    def __init__(self, *, movable, sd, cifinish_out=None, overwrite_saves=False, skip_contents=False,
+                 boot9=None, seeddb=None):
         self.event = Events()
         self.log_lines = []  # Stores all info messages for user to view
 
         self.crypto = CryptoEngine(boot9=boot9)
         self.crypto.setup_sd_key_from_file(movable)
         self.seeddb = seeddb
-        self.readers: 'List[Union[CDNReader, CIAReader]]' = []
+        self.readers: 'List[Tuple[Union[CDNReader, CIAReader], Union[PathLike, bytes, str]]]' = []
         self.sd = sd
         self.skip_contents = skip_contents
         self.overwrite_saves = overwrite_saves
@@ -249,12 +261,12 @@ class CustomInstall:
             if reader.tmd.title_id.startswith('00048'):  # DSiWare
                 self.log(f'Skipping {reader.tmd.title_id} - DSiWare is not supported')
                 continue
-            readers.append(reader)
+            readers.append((reader, path))
         self.readers = readers
 
     def check_size(self):
         total_size = 0
-        for r in self.readers:
+        for r, _ in self.readers:
             total_size += get_install_size(r)
 
         free_space = get_free_space(self.sd)
@@ -269,7 +281,7 @@ class CustomInstall:
             save3ds_fuse_path += '.exe'
         if not isfile(save3ds_fuse_path):
             self.log("Couldn't find " + save3ds_fuse_path, 2)
-            return None, False
+            return None, False, 0
 
         crypto = self.crypto
         # TODO: Move a lot of these into their own methods
@@ -280,6 +292,8 @@ class CustomInstall:
                               f'please remove extra directories')
         elif len(id1s) == 0:
             raise SDPathError(f'Could not find a suitable id1 directory for id0 {crypto.id0.hex()}')
+        id1 = id1s[0]
+        sd_path = join(sd_path, id1)
 
         if self.cifinish_out:
             cifinish_path = self.cifinish_out
@@ -294,7 +308,41 @@ class CustomInstall:
                      f'This could mean an issue with the SD card or the filesystem. Please check it for errors.\n'
                      f'It is also possible, though less likely, to be an issue with custom-install.\n'
                      f'Exiting now to prevent possible issues. If you want to try again, delete cifinish.bin from the SD card and re-run custom-install.')
-            return None, False
+            return None, False, 0
+
+        db_path = join(sd_path, 'dbs')
+        titledb_path = join(db_path, 'title.db')
+        importdb_path = join(db_path, 'import.db')
+        if not isfile(titledb_path):
+            makedirs(db_path, exist_ok=True)
+            with gzip.open(join(script_dir, 'title.db.gz')) as f:
+                tdb = f.read()
+
+            self.log(f'Creating title.db...')
+            with open(titledb_path, 'wb') as o:
+                with self.crypto.create_ctr_io(Keyslot.SD, o, self.crypto.sd_path_to_iv('/dbs/title.db')) as e:
+                    e.write(tdb)
+
+                    cmac = crypto.create_cmac_object(Keyslot.CMACSDNAND)
+                    cmac_data = [b'CTR-9DB0', 0x2.to_bytes(4, 'little'), tdb[0x100:0x200]]
+                    cmac.update(sha256(b''.join(cmac_data)).digest())
+
+                    e.seek(0)
+                    e.write(cmac.digest())
+
+            self.log(f'Creating import.db...')
+            with open(importdb_path, 'wb') as o:
+                with self.crypto.create_ctr_io(Keyslot.SD, o, self.crypto.sd_path_to_iv('/dbs/import.db')) as e:
+                    e.write(tdb)
+
+                    cmac = crypto.create_cmac_object(Keyslot.CMACSDNAND)
+                    cmac_data = [b'CTR-9DB0', 0x3.to_bytes(4, 'little'), tdb[0x100:0x200]]
+                    cmac.update(sha256(b''.join(cmac_data)).digest())
+
+                    e.seek(0)
+                    e.write(cmac.digest())
+
+            del tdb
 
         with TemporaryDirectory(suffix='-custom-install') as tempdir:
             # set up the common arguments for the two times we call save3ds_fuse
@@ -325,18 +373,19 @@ class CustomInstall:
                 self.log('Command line:')
                 for l in pformat(out.args).split('\n'):
                     self.log(l)
-                return None, False
+                return None, False, 0
 
-            sd_path = join(sd_path, id1s[0])
-
-            load_seeddb(self.seeddb)
+            if self.seeddb:
+                load_seeddb(self.seeddb)
 
             install_state = {'installed': [], 'failed': []}
 
             # Now loop through all provided cia files
-            for idx, cia in enumerate(self.readers):
+            for idx, info in enumerate(self.readers):
+                cia, path = info
 
                 self.event.on_cia_start(idx)
+                self.event.update_status(path, InstallStatus.Starting)
 
                 temp_title_root = join(self.sd, f'ci-install-temp-{cia.tmd.title_id}-{randint(0, 0xFFFFFFFF):08x}')
                 makedirs(temp_title_root, exist_ok=True)
@@ -383,6 +432,7 @@ class CustomInstall:
                 temp_content_root = join(temp_title_root, 'content')
 
                 if not self.skip_contents:
+                    self.event.update_status(path, InstallStatus.Writing)
                     makedirs(join(temp_content_root, 'cmd'), exist_ok=True)
                     if cia.tmd.save_size:
                         makedirs(join(temp_title_root, 'data'), exist_ok=True)
@@ -423,6 +473,7 @@ class CustomInstall:
                                 install_state['failed'].append(display_title)
                                 rename(temp_title_root, temp_title_root + '-corrupted')
                                 do_continue = True
+                                self.event.update_status(path, InstallStatus.Failed)
                                 break
 
                     if do_continue:
@@ -534,6 +585,7 @@ class CustomInstall:
                     b'\0' * 0x2c
                 ]
 
+                self.event.update_status(path, InstallStatus.Finishing)
                 if isdir(title_root):
                     self.log(f'Removing original install at {title_root}...')
                     rmtree(title_root)
@@ -563,11 +615,19 @@ class CustomInstall:
                     for l in pformat(out.args).split('\n'):
                         self.log(l)
                     install_state['failed'].append(display_title)
-
-                install_state['installed'].append(display_title)
+                    self.event.update_status(path, InstallStatus.Failed)
+                else:
+                    install_state['installed'].append(display_title)
+                    self.event.update_status(path, InstallStatus.Done)
 
             copied = False
+            # launchable applications, not DLC or update data
+            application_count = len(glob(join(tempdir, '00040000*')))
             if install_state['installed']:
+                if application_count >= 300:
+                    self.log(f'{application_count} installed applications were detected.', 1)
+                    self.log('The HOME Menu will only show 300 icons.', 1)
+                    self.log('Some applications (not updates or DLC) will need to be deleted.', 1)
                 finalize_3dsx_orig_path = join(script_dir, 'custom-install-finalize.3dsx')
                 hb_dir = join(self.sd, '3ds')
                 finalize_3dsx_path = join(hb_dir, 'custom-install-finalize.3dsx')
@@ -583,7 +643,7 @@ class CustomInstall:
                 if copied:
                     self.log('custom-install-finalize has been copied to the SD card.')
 
-            return install_state, copied
+            return install_state, copied, application_count
 
     def get_sd_path(self):
         sd_path = join(self.sd, 'Nintendo 3DS', self.crypto.id0.hex())

make-standalone.bat

@@ -6,6 +6,7 @@ copy TaskbarLib.tlb build\custom-install-standalone
 copy bin\win32\save3ds_fuse.exe build\custom-install-standalone\bin
 copy bin\README build\custom-install-standalone\bin
 copy custom-install-finalize.3dsx build\custom-install-standalone
+copy title.db.gz build\custom-install-standalone
 copy extras\windows-quickstart.txt build\custom-install-standalone
 copy LICENSE.md build\custom-install-standalone
 python -m zipfile -c dist\custom-install-standalone.zip build\custom-install-standalone

requirements-win32.txt

@@ -1,2 +1,2 @@
 -r requirements.txt
-comtypes==1.1.7
+comtypes==1.1.8

requirements.txt

@@ -1,2 +1,2 @@
 events==0.4
-pyctr==0.4.5
+pyctr==0.4.6