version 2.1b4 & finalize 1.6

Added checks for existing Ticket and entry in Title Database

.gitignore

@@ -18,3 +18,6 @@ venv/
 =======
 
 *.pyc
+/build/
+/dist/
+/custom-install-finalize.3dsx

LICENSE.md

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2019 Ian Burgwin
+Copyright (c) 2019-2021 Ian Burgwin
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,7 +1,7 @@
 [![License](https://img.shields.io/badge/License-MIT-blue.svg)]() ![Releases](https://img.shields.io/github/downloads/ihaveamac/custom-install/total.svg)
 
 # custom-install
-Experimental script to automate the process of a manual title install for Nintendo 3DS. Originally created late June 2019.
+Installs a title directly to an SD card for the Nintendo 3DS. Originally created late June 2019.
 
 ## Summary
 

ci-gui.py

@@ -5,7 +5,7 @@
 # You can find the full license text in LICENSE.md in the root of this project.
 
 from os import environ, scandir
-from os.path import abspath, join, isfile, dirname
+from os.path import abspath, basename, dirname, join, isfile
 from sys import exc_info, platform
 from threading import Thread, Lock
 from time import strftime
@@ -16,13 +16,18 @@ import tkinter.filedialog as fd
 import tkinter.messagebox as mb
 from typing import TYPE_CHECKING
 
+from pyctr.crypto import MissingSeedError, CryptoEngine, load_seeddb
 from pyctr.crypto.engine import b9_paths
 from pyctr.util import config_dirs
+from pyctr.type.cdn import CDNError
+from pyctr.type.cia import CIAError
+from pyctr.type.tmd import TitleMetadataError
 
-from custominstall import CustomInstall
+from custominstall import CustomInstall, CI_VERSION, load_cifinish, InvalidCIFinishError, InstallStatus
 
 if TYPE_CHECKING:
-    from typing import List
+    from os import PathLike
+    from typing import Dict, List, Union
 
 is_windows = platform == 'win32'
 taskbar = None
@@ -37,7 +42,7 @@ if is_windows:
     except ModuleNotFoundError:
         pass
 
-file_parent = dirname(__file__)
+file_parent = dirname(abspath(__file__))
 
 # automatically load boot9 if it's in the current directory
 b9_paths.insert(0, join(file_parent, 'boot9.bin'))
@@ -52,6 +57,10 @@ except KeyError:
 seeddb_paths.insert(0, join(file_parent, 'seeddb.bin'))
 
 
+def clamp(n, smallest, largest):
+    return max(smallest, min(n, largest))
+
+
 def find_first_file(paths):
     for p in paths:
         if isfile(p):
@@ -63,6 +72,18 @@ default_b9_path = find_first_file(b9_paths)
 default_seeddb_path = find_first_file(seeddb_paths)
 default_movable_sed_path = find_first_file([join(file_parent, 'movable.sed')])
 
+if default_seeddb_path:
+    load_seeddb(default_seeddb_path)
+
+statuses = {
+    InstallStatus.Waiting: 'Waiting',
+    InstallStatus.Starting: 'Starting',
+    InstallStatus.Writing: 'Writing',
+    InstallStatus.Finishing: 'Finishing',
+    InstallStatus.Done: 'Done',
+    InstallStatus.Failed: 'Failed',
+}
+
 
 class ConsoleFrame(ttk.Frame):
     def __init__(self, parent: tk.BaseWidget = None, starting_lines: 'List[str]' = None):
@@ -94,13 +115,155 @@ class ConsoleFrame(ttk.Frame):
         self.text.configure(state=tk.DISABLED)
 
 
+def simple_listbox_frame(parent, title: 'str', items: 'List[str]'):
+    frame = ttk.LabelFrame(parent, text=title)
+    frame.rowconfigure(0, weight=1)
+    frame.columnconfigure(0, weight=1)
+
+    scrollbar = ttk.Scrollbar(frame, orient=tk.VERTICAL)
+    scrollbar.grid(row=0, column=1, sticky=tk.NSEW)
+
+    box = tk.Listbox(frame, highlightthickness=0, yscrollcommand=scrollbar.set, selectmode=tk.EXTENDED)
+    box.grid(row=0, column=0, sticky=tk.NSEW)
+    scrollbar.config(command=box.yview)
+
+    box.insert(tk.END, *items)
+
+    box.config(height=clamp(len(items), 3, 10))
+
+    return frame
+
+
+class TitleReadFailResults(tk.Toplevel):
+    def __init__(self, parent: tk.Tk = None, *, failed: 'Dict[str, str]'):
+        super().__init__(parent)
+        self.parent = parent
+
+        self.wm_withdraw()
+        self.wm_transient(self.parent)
+        self.grab_set()
+        self.wm_title('Failed to add titles')
+
+        self.rowconfigure(0, weight=1)
+        self.columnconfigure(0, weight=1)
+
+        outer_container = ttk.Frame(self)
+        outer_container.grid(sticky=tk.NSEW)
+        outer_container.rowconfigure(0, weight=0)
+        outer_container.rowconfigure(1, weight=1)
+        outer_container.columnconfigure(0, weight=1)
+
+        message_label = ttk.Label(outer_container, text="Some titles couldn't be added.")
+        message_label.grid(row=0, column=0, sticky=tk.NSEW, padx=10, pady=10)
+
+        treeview_frame = ttk.Frame(outer_container)
+        treeview_frame.grid(row=1, column=0, sticky=tk.NSEW)
+        treeview_frame.rowconfigure(0, weight=1)
+        treeview_frame.columnconfigure(0, weight=1)
+
+        treeview_scrollbar = ttk.Scrollbar(treeview_frame, orient=tk.VERTICAL)
+        treeview_scrollbar.grid(row=0, column=1, sticky=tk.NSEW)
+
+        treeview = ttk.Treeview(treeview_frame, yscrollcommand=treeview_scrollbar.set)
+        treeview.grid(row=0, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+        treeview.configure(columns=('filepath', 'reason'), show='headings')
+
+        treeview.column('filepath', width=200, anchor=tk.W)
+        treeview.heading('filepath', text='File path')
+        treeview.column('reason', width=400, anchor=tk.W)
+        treeview.heading('reason', text='Reason')
+
+        treeview_scrollbar.configure(command=treeview.yview)
+
+        for path, reason in failed.items():
+            treeview.insert('', tk.END, text=path, iid=path, values=(basename(path), reason))
+
+        ok_frame = ttk.Frame(outer_container)
+        ok_frame.grid(row=2, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+        ok_frame.rowconfigure(0, weight=1)
+        ok_frame.columnconfigure(0, weight=1)
+
+        ok_button = ttk.Button(ok_frame, text='OK', command=self.destroy)
+        ok_button.grid(row=0, column=0)
+
+        self.wm_deiconify()
+
+
+class InstallResults(tk.Toplevel):
+    def __init__(self, parent: tk.Tk = None, *, install_state: 'Dict[str, List[str]]', copied_3dsx: bool,
+                 application_count: int):
+        super().__init__(parent)
+        self.parent = parent
+
+        self.wm_withdraw()
+        self.wm_transient(self.parent)
+        self.grab_set()
+        self.wm_title('Install results')
+
+        self.rowconfigure(0, weight=1)
+        self.columnconfigure(0, weight=1)
+
+        outer_container = ttk.Frame(self)
+        outer_container.grid(sticky=tk.NSEW)
+        outer_container.rowconfigure(0, weight=0)
+        outer_container.columnconfigure(0, weight=1)
+
+        if install_state['failed'] and install_state['installed']:
+            # some failed and some worked
+            message = ('Some titles were installed, some failed. Please check the output for more details.\n'
+                       'The ones that were installed can be finished with custom-install-finalize.')
+        elif install_state['failed'] and not install_state['installed']:
+            # all failed
+            message = 'All titles failed to install. Please check the output for more details.'
+        elif install_state['installed'] and not install_state['failed']:
+            # all worked
+            message = 'All titles were installed.'
+        else:
+            message = 'Nothing was installed.'
+
+        if install_state['installed'] and copied_3dsx:
+            message += '\n\ncustom-install-finalize has been copied to the SD card.'
+
+        if application_count >= 300:
+            message += (f'\n\nWarning: {application_count} installed applications were detected.\n'
+                        f'The HOME Menu will only show 300 icons.\n'
+                        f'Some applications (not updates or DLC) will need to be deleted.')
+
+        message_label = ttk.Label(outer_container, text=message)
+        message_label.grid(row=0, column=0, sticky=tk.NSEW, padx=10, pady=10)
+
+        if install_state['installed']:
+            outer_container.rowconfigure(1, weight=1)
+            frame = simple_listbox_frame(outer_container, 'Installed', install_state['installed'])
+            frame.grid(row=1, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+
+        if install_state['failed']:
+            outer_container.rowconfigure(2, weight=1)
+            frame = simple_listbox_frame(outer_container, 'Failed', install_state['failed'])
+            frame.grid(row=2, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+
+        ok_frame = ttk.Frame(outer_container)
+        ok_frame.grid(row=3, column=0, sticky=tk.NSEW, padx=10, pady=(0, 10))
+        ok_frame.rowconfigure(0, weight=1)
+        ok_frame.columnconfigure(0, weight=1)
+
+        ok_button = ttk.Button(ok_frame, text='OK', command=self.destroy)
+        ok_button.grid(row=0, column=0)
+
+        self.wm_deiconify()
+
+
 class CustomInstallGUI(ttk.Frame):
     console = None
+    b9_loaded = False
 
     def __init__(self, parent: tk.Tk = None):
         super().__init__(parent)
         self.parent = parent
 
+        # readers to give to CustomInstall at the install
+        self.readers = {}
+
         self.lock = Lock()
 
         self.log_messages = []
@@ -130,15 +293,27 @@ class CustomInstallGUI(ttk.Frame):
             f = fd.askdirectory(parent=parent, title='Select SD root (the directory or drive that contains '
                                                      '"Nintendo 3DS")', initialdir=file_parent, mustexist=True)
             if f:
+                cifinish_path = join(f, 'cifinish.bin')
+                try:
+                    load_cifinish(cifinish_path)
+                except InvalidCIFinishError:
+                    self.show_error(f'{cifinish_path} was corrupt!\n\n'
+                                    f'This could mean an issue with the SD card or the filesystem. Please check it for errors.\n'
+                                    f'It is also possible, though less likely, to be an issue with custom-install.\n\n'
+                                    f'Stopping now to prevent possible issues. If you want to try again, delete cifinish.bin from the SD card and re-run custom-install.')
+                    return
+
                 sd_selected.delete('1.0', tk.END)
                 sd_selected.insert(tk.END, f)
 
-                sd_msed_path = find_first_file([join(f, 'gm9', 'out', 'movable.sed'), join(f, 'movable.sed')])
-                if sd_msed_path:
-                    self.log('Found movable.sed on SD card at ' + sd_msed_path)
-                    box = self.file_picker_textboxes['movable.sed']
-                    box.delete('1.0', tk.END)
-                    box.insert(tk.END, sd_msed_path)
+                for filename in ['boot9.bin', 'seeddb.bin', 'movable.sed']:
+                    path = auto_input_filename(self, f, filename)
+                    if filename == 'boot9.bin':
+                        self.check_b9_loaded()
+                        self.enable_buttons()
+                    if filename == 'seeddb.bin':
+                        load_seeddb(path)
+
 
         sd_type_label = ttk.Label(file_pickers, text='SD root')
         sd_type_label.grid(row=0, column=0)
@@ -151,14 +326,25 @@ class CustomInstallGUI(ttk.Frame):
 
         self.file_picker_textboxes['sd'] = sd_selected
 
+        def auto_input_filename(self, f, filename):
+            sd_msed_path = find_first_file([join(f, 'gm9', 'out', filename), join(f, filename)])
+            if sd_msed_path:
+                self.log('Found ' + filename + ' on SD card at ' + sd_msed_path)
+                if filename.endswith('bin'):
+                    filename = filename.split('.')[0]
+                box = self.file_picker_textboxes[filename]
+                box.delete('1.0', tk.END)
+                box.insert(tk.END, sd_msed_path)
+                return sd_msed_path
         # This feels so wrong.
-        def create_required_file_picker(type_name, types, default, row):
+        def create_required_file_picker(type_name, types, default, row, callback=lambda filename: None):
             def internal_callback():
                 f = fd.askopenfilename(parent=parent, title='Select ' + type_name, filetypes=types,
                                        initialdir=file_parent)
                 if f:
                     selected.delete('1.0', tk.END)
                     selected.insert(tk.END, f)
+                    callback(f)
 
             type_label = ttk.Label(file_pickers, text=type_name)
             type_label.grid(row=row, column=0)
@@ -173,59 +359,104 @@ class CustomInstallGUI(ttk.Frame):
 
             self.file_picker_textboxes[type_name] = selected
 
-        create_required_file_picker('boot9', [('boot9 file', '*.bin')], default_b9_path, 1)
-        create_required_file_picker('seeddb', [('seeddb file', '*.bin')], default_seeddb_path, 2)
+        def b9_callback(path: 'Union[PathLike, bytes, str]'):
+            self.check_b9_loaded()
+            self.enable_buttons()
+
+        def seeddb_callback(path: 'Union[PathLike, bytes, str]'):
+            load_seeddb(path)
+
+        create_required_file_picker('boot9', [('boot9 file', '*.bin')], default_b9_path, 1, b9_callback)
+        create_required_file_picker('seeddb', [('seeddb file', '*.bin')], default_seeddb_path, 2, seeddb_callback)
         create_required_file_picker('movable.sed', [('movable.sed file', '*.sed')], default_movable_sed_path, 3)
 
         # ---------------------------------------------------------------- #
         # create buttons to add cias
-        listbox_buttons = ttk.Frame(self)
-        listbox_buttons.grid(row=1, column=0)
+        titlelist_buttons = ttk.Frame(self)
+        titlelist_buttons.grid(row=1, column=0)
 
         def add_cias_callback():
             files = fd.askopenfilenames(parent=parent, title='Select CIA files', filetypes=[('CIA files', '*.cia')],
                                         initialdir=file_parent)
+            results = {}
             for f in files:
-                self.add_cia(f)
+                success, reason = self.add_cia(f)
+                if not success:
+                    results[f] = reason
 
-        add_cias = ttk.Button(listbox_buttons, text='Add CIAs', command=add_cias_callback)
+            if results:
+                title_read_fail_window = TitleReadFailResults(self.parent, failed=results)
+                title_read_fail_window.focus()
+            self.sort_treeview()
+
+        add_cias = ttk.Button(titlelist_buttons, text='Add CIAs', command=add_cias_callback)
         add_cias.grid(row=0, column=0)
 
+        def add_cdn_callback():
+            d = fd.askdirectory(parent=parent, title='Select folder containing title contents in CDN format',
+                                initialdir=file_parent)
+            if d:
+                if isfile(join(d, 'tmd')):
+                    success, reason = self.add_cia(d)
+                    if not success:
+                        self.show_error(f"Couldn't add {basename(d)}: {reason}")
+                    else:
+                        self.sort_treeview()
+                else:
+                    self.show_error('tmd file not found in the CDN directory:\n' + d)
+
+        add_cdn = ttk.Button(titlelist_buttons, text='Add CDN title folder', command=add_cdn_callback)
+        add_cdn.grid(row=0, column=1)
+
         def add_dirs_callback():
             d = fd.askdirectory(parent=parent, title='Select folder containing CIA files', initialdir=file_parent)
             if d:
+                results = {}
                 for f in scandir(d):
                     if f.name.lower().endswith('.cia'):
-                        self.add_cia(f.path)
+                        success, reason = self.add_cia(f.path)
+                        if not success:
+                            results[f] = reason
 
-        add_dirs = ttk.Button(listbox_buttons, text='Add folder', command=add_dirs_callback)
-        add_dirs.grid(row=0, column=1)
+                if results:
+                    title_read_fail_window = TitleReadFailResults(self.parent, failed=results)
+                    title_read_fail_window.focus()
+                self.sort_treeview()
+
+        add_dirs = ttk.Button(titlelist_buttons, text='Add folder', command=add_dirs_callback)
+        add_dirs.grid(row=0, column=2)
 
         def remove_selected_callback():
-            indexes = self.cia_listbox.curselection()
-            n = 0
-            for i in indexes:
-                self.cia_listbox.delete(i - n)
-                n += 1
+            for entry in self.treeview.selection():
+                self.remove_cia(entry)
 
-        remove_selected = ttk.Button(listbox_buttons, text='Remove selected', command=remove_selected_callback)
-        remove_selected.grid(row=0, column=2)
+        remove_selected = ttk.Button(titlelist_buttons, text='Remove selected', command=remove_selected_callback)
+        remove_selected.grid(row=0, column=3)
 
         # ---------------------------------------------------------------- #
-        # create listbox
-        listbox_frame = ttk.Frame(self)
-        listbox_frame.grid(row=2, column=0, sticky=tk.NSEW)
-        listbox_frame.rowconfigure(0, weight=1)
-        listbox_frame.columnconfigure(0, weight=1)
+        # create treeview
+        treeview_frame = ttk.Frame(self)
+        treeview_frame.grid(row=2, column=0, sticky=tk.NSEW)
+        treeview_frame.rowconfigure(0, weight=1)
+        treeview_frame.columnconfigure(0, weight=1)
 
-        cia_listbox_scrollbar = ttk.Scrollbar(listbox_frame, orient=tk.VERTICAL)
-        cia_listbox_scrollbar.grid(row=0, column=1, sticky=tk.NSEW)
+        treeview_scrollbar = ttk.Scrollbar(treeview_frame, orient=tk.VERTICAL)
+        treeview_scrollbar.grid(row=0, column=1, sticky=tk.NSEW)
 
-        self.cia_listbox = tk.Listbox(listbox_frame, highlightthickness=0, yscrollcommand=cia_listbox_scrollbar.set,
-                                      selectmode=tk.EXTENDED)
-        self.cia_listbox.grid(row=0, column=0, sticky=tk.NSEW)
+        self.treeview = ttk.Treeview(treeview_frame, yscrollcommand=treeview_scrollbar.set)
+        self.treeview.grid(row=0, column=0, sticky=tk.NSEW)
+        self.treeview.configure(columns=('filepath', 'titleid', 'titlename', 'status'), show='headings')
 
-        cia_listbox_scrollbar.config(command=self.cia_listbox.yview)
+        self.treeview.column('filepath', width=200, anchor=tk.W)
+        self.treeview.heading('filepath', text='File path')
+        self.treeview.column('titleid', width=70, anchor=tk.W)
+        self.treeview.heading('titleid', text='Title ID')
+        self.treeview.column('titlename', width=150, anchor=tk.W)
+        self.treeview.heading('titlename', text='Title name')
+        self.treeview.column('status', width=20, anchor=tk.W)
+        self.treeview.heading('status', text='Status')
+
+        treeview_scrollbar.configure(command=self.treeview.yview)
 
         # ---------------------------------------------------------------- #
         # create progressbar
@@ -258,8 +489,7 @@ class CustomInstallGUI(ttk.Frame):
         self.status_label = ttk.Label(self, text='Waiting...')
         self.status_label.grid(row=5, column=0, sticky=tk.NSEW)
 
-        self.log('custom-install by ihaveamac', status=False)
-        self.log('https://github.com/ihaveamac/custom-install', status=False)
+        self.log(f'custom-install {CI_VERSION} - https://github.com/ihaveamac/custom-install', status=False)
 
         if is_windows and not taskbar:
             self.log('Note: comtypes module not found.')
@@ -267,11 +497,65 @@ class CustomInstallGUI(ttk.Frame):
 
         self.log('Ready.')
 
-        self.disable_during_install = (add_cias, add_dirs, remove_selected, start, *self.file_picker_textboxes.values())
+        self.require_boot9 = (add_cias, add_cdn, add_dirs, remove_selected, start)
+
+        self.disable_buttons()
+        self.check_b9_loaded()
+        self.enable_buttons()
+        if not self.b9_loaded:
+            self.log('Note: boot9 was not auto-detected. Please choose it before adding any titles.')
+
+    def sort_treeview(self):
+        l = [(self.treeview.set(k, 'titlename'), k) for k in self.treeview.get_children()]
+        # sort by title name
+        l.sort(key=lambda x: x[0].lower())
+
+        for idx, pair in enumerate(l):
+            self.treeview.move(pair[1], '', idx)
+
+    def check_b9_loaded(self):
+        if not self.b9_loaded:
+            boot9 = self.file_picker_textboxes['boot9'].get('1.0', tk.END).strip()
+            try:
+                tmp_crypto = CryptoEngine(boot9=boot9)
+                self.b9_loaded = tmp_crypto.b9_keys_set
+            except:
+                return False
+        return self.b9_loaded
+
+    def update_status(self, path: 'Union[PathLike, bytes, str]', status: InstallStatus):
+        self.treeview.set(path, 'status', statuses[status])
 
     def add_cia(self, path):
+        if not self.check_b9_loaded():
+            # this shouldn't happen
+            return False, 'Please choose boot9 first'
         path = abspath(path)
-        self.cia_listbox.insert(tk.END, path)
+        if path in self.readers:
+            return False, 'File already in list'
+        try:
+            reader = CustomInstall.get_reader(path)
+        except (CIAError, CDNError, TitleMetadataError):
+            return False, 'Failed to read as a CIA or CDN title, probably corrupt'
+        except MissingSeedError:
+            return False, 'Latest seeddb.bin is required, check the README for details'
+        except Exception as e:
+            return False, f'Exception occurred: {type(e).__name__}: {e}'
+
+        if reader.tmd.title_id.startswith('00048'):
+            return False, 'DSiWare is not supported'
+        try:
+            title_name = reader.contents[0].exefs.icon.get_app_title().short_desc
+        except:
+            title_name = '(No title)'
+        self.treeview.insert('', tk.END, text=path, iid=path,
+                             values=(path, reader.tmd.title_id, title_name, statuses[InstallStatus.Waiting]))
+        self.readers[path] = reader
+        return True, ''
+
+    def remove_cia(self, path):
+        self.treeview.delete(path)
+        del self.readers[path]
 
     def open_console(self):
         if self.console:
@@ -316,25 +600,26 @@ class CustomInstallGUI(ttk.Frame):
         mb.showinfo('Info', message, parent=self.parent)
 
     def disable_buttons(self):
-        for b in self.disable_during_install:
+        for b in self.require_boot9:
+            b.config(state=tk.DISABLED)
+        for b in self.file_picker_textboxes.values():
             b.config(state=tk.DISABLED)
 
     def enable_buttons(self):
-        for b in self.disable_during_install:
+        if self.b9_loaded:
+            for b in self.require_boot9:
+                b.config(state=tk.NORMAL)
+        for b in self.file_picker_textboxes.values():
             b.config(state=tk.NORMAL)
 
     def start_install(self):
         sd_root = self.file_picker_textboxes['sd'].get('1.0', tk.END).strip()
-        boot9 = self.file_picker_textboxes['boot9'].get('1.0', tk.END).strip()
         seeddb = self.file_picker_textboxes['seeddb'].get('1.0', tk.END).strip()
         movable_sed = self.file_picker_textboxes['movable.sed'].get('1.0', tk.END).strip()
 
         if not sd_root:
             self.show_error('SD root is not specified.')
             return
-        if not boot9:
-            self.show_error('boot9 is not specified.')
-            return
         if not movable_sed:
             self.show_error('movable.sed is not specified.')
             return
@@ -344,24 +629,33 @@ class CustomInstallGUI(ttk.Frame):
                                     'Continue?'):
                 return
 
-        self.disable_buttons()
-        self.log('Starting install...')
-
-        cias = self.cia_listbox.get(0, tk.END)
-        if not len(cias):
+        if not len(self.readers):
             self.show_error('There are no titles added to install.')
             return
 
-        installer = CustomInstall(boot9=boot9,
-                                  seeddb=seeddb,
-                                  movable=movable_sed,
-                                  cias=cias,
+        for path in self.readers.keys():
+            self.update_status(path, InstallStatus.Waiting)
+        self.disable_buttons()
+        self.log('Starting install...')
+
+        if taskbar:
+            taskbar.SetProgressState(self.hwnd, tbl.TBPF_NORMAL)
+
+        installer = CustomInstall(movable=movable_sed,
                                   sd=sd_root,
                                   skip_contents=self.skip_contents_var.get() == 1,
                                   overwrite_saves=self.overwrite_saves_var.get() == 1)
 
+        # use the treeview which has been sorted alphabetically
+        readers_final = []
+        for k in self.treeview.get_children():
+            filepath = self.treeview.set(k, 'filepath')
+            readers_final.append((self.readers[filepath], filepath))
+
+        installer.readers = readers_final
+
         finished_percent = 0
-        max_percentage = 100 * len(cias)
+        max_percentage = 100 * len(self.readers)
         self.progressbar.config(maximum=max_percentage)
 
         def ci_on_log_msg(message, *args, **kwargs):
@@ -392,24 +686,29 @@ class CustomInstallGUI(ttk.Frame):
         installer.event.update_percentage += ci_update_percentage
         installer.event.on_error += ci_on_error
         installer.event.on_cia_start += ci_on_cia_start
+        installer.event.update_status += self.update_status
 
-        if taskbar:
-            taskbar.SetProgressState(self.hwnd, tbl.TBPF_NORMAL)
+        if self.skip_contents_var.get() != 1:
+            total_size, free_space = installer.check_size()
+            if total_size > free_space:
+                self.show_error(f'Not enough free space.\n'
+                                f'Combined title install size: {total_size / (1024 * 1024):0.2f} MiB\n'
+                                f'Free space: {free_space / (1024 * 1024):0.2f} MiB')
+                self.enable_buttons()
+                return
 
         def install():
             try:
-                result, copied_3dsx = installer.start(continue_on_fail=False)
-                if result is True:
-                    self.log('Done!')
-                    if copied_3dsx:
-                        self.show_info('custom-install-finalize has been copied to the SD card.\n'
-                                       'To finish the install, run this on the console through the homebrew launcher.\n'
-                                       'This will install a ticket and seed if required.')
-                    else:
-                        self.show_info('To finish the install, run custom-install-finalize on the console.\n'
-                                       'This will install a ticket and seed if required.')
-                elif result is False:
-                    self.show_error('An error occurred when trying to run save3ds_fuse.')
+                result, copied_3dsx, application_count = installer.start()
+                if result:
+                    result_window = InstallResults(self.parent,
+                                                   install_state=result,
+                                                   copied_3dsx=copied_3dsx,
+                                                   application_count=application_count)
+                    result_window.focus()
+                elif result is None:
+                    self.show_error("An error occurred when trying to run save3ds_fuse.\n"
+                                    "Either title.db doesn't exist, or save3ds_fuse couldn't be run.")
                     self.open_console()
             except:
                 installer.event.on_error(exc_info())
@@ -420,7 +719,7 @@ class CustomInstallGUI(ttk.Frame):
 
 
 window = tk.Tk()
-window.title('custom-install')
+window.title(f'custom-install {CI_VERSION}')
 frame = CustomInstallGUI(window)
 frame.pack(fill=tk.BOTH, expand=True)
 window.mainloop()

custominstall.py

@@ -5,13 +5,17 @@
 # You can find the full license text in LICENSE.md in the root of this project.
 
 from argparse import ArgumentParser
-from os import makedirs, scandir
-from os.path import dirname, join, isfile
+from enum import Enum
+from glob import glob
+import gzip
+from os import makedirs, rename, scandir
+from os.path import dirname, join, isdir, isfile
 from random import randint
 from hashlib import sha256
-from locale import getpreferredencoding
-from shutil import copyfile
+from pprint import pformat
+from shutil import copyfile, copy2, rmtree
 import sys
+from sys import platform, executable
 from tempfile import TemporaryDirectory
 from traceback import format_exception
 from typing import BinaryIO, TYPE_CHECKING
@@ -19,22 +23,34 @@ import subprocess
 
 if TYPE_CHECKING:
     from os import PathLike
-    from typing import Union
+    from typing import List, Union, Tuple
 
 from events import Events
 
 from pyctr.crypto import CryptoEngine, Keyslot, load_seeddb, get_seed
-from pyctr.type.cia import CIAReader, CIASection
+from pyctr.type.cdn import CDNReader, CDNError
+from pyctr.type.cia import CIAReader, CIAError
 from pyctr.type.ncch import NCCHSection
+from pyctr.type.tmd import TitleMetadataError
 from pyctr.util import roundup
 
-is_windows = sys.platform == 'win32'
+if platform == 'msys':
+    platform = 'win32'
+
+is_windows = platform == 'win32'
+
+if is_windows:
+    from ctypes import c_wchar_p, pointer, c_ulonglong, windll
+else:
+    from os import statvfs
+
+CI_VERSION = '2.1b4'
 
 # used to run the save3ds_fuse binary next to the script
 frozen = getattr(sys, 'frozen', False)
 script_dir: str
 if frozen:
-    script_dir = dirname(sys.executable)
+    script_dir = dirname(executable)
 else:
     script_dir = dirname(__file__)
 
@@ -60,6 +76,35 @@ class InvalidCIFinishError(Exception):
     pass
 
 
+class InstallStatus(Enum):
+    Waiting = 0
+    Starting = 1
+    Writing = 2
+    Finishing = 3
+    Done = 4
+    Failed = 5
+
+
+def get_free_space(path: 'Union[PathLike, bytes, str]'):
+    if is_windows:
+        lpSectorsPerCluster = c_ulonglong(0)
+        lpBytesPerSector = c_ulonglong(0)
+        lpNumberOfFreeClusters = c_ulonglong(0)
+        lpTotalNumberOfClusters = c_ulonglong(0)
+        ret = windll.kernel32.GetDiskFreeSpaceW(c_wchar_p(path), pointer(lpSectorsPerCluster),
+                                                pointer(lpBytesPerSector),
+                                                pointer(lpNumberOfFreeClusters),
+                                                pointer(lpTotalNumberOfClusters))
+        if not ret:
+            raise WindowsError
+        free_blocks = lpNumberOfFreeClusters.value * lpSectorsPerCluster.value
+        free_bytes = free_blocks * lpBytesPerSector.value
+    else:
+        stv = statvfs(path)
+        free_bytes = stv.f_bavail * stv.f_frsize
+    return free_bytes
+
+
 def load_cifinish(path: 'Union[PathLike, bytes, str]'):
     try:
         with open(path, 'rb') as f:
@@ -140,19 +185,32 @@ def save_cifinish(path: 'Union[PathLike, bytes, str]', data: dict):
             out.write(b''.join(finalize_entry_data))
 
 
+def get_install_size(title: 'Union[CIAReader, CDNReader]'):
+    sizes = [1] * 5
+
+    if title.tmd.save_size:
+        # one for the data directory, one for the 00000001.sav file
+        sizes.extend((1, title.tmd.save_size))
+
+    for record in title.content_info:
+        sizes.append(record.size)
+
+    # this calculates the size to put in the Title Info Entry
+    title_size = sum(roundup(x, TITLE_ALIGN_SIZE) for x in sizes)
+
+    return title_size
+
+
 class CustomInstall:
-
-    cia: CIAReader
-
-    def __init__(self, boot9, seeddb, movable, cias, sd, cifinish_out=None,
-                 overwrite_saves=False, skip_contents=False):
+    def __init__(self, *, movable, sd, cifinish_out=None, overwrite_saves=False, skip_contents=False,
+                 boot9=None, seeddb=None):
         self.event = Events()
         self.log_lines = []  # Stores all info messages for user to view
 
         self.crypto = CryptoEngine(boot9=boot9)
         self.crypto.setup_sd_key_from_file(movable)
         self.seeddb = seeddb
-        self.cias = cias
+        self.readers: 'List[Tuple[Union[CDNReader, CIAReader], Union[PathLike, bytes, str]]]' = []
         self.sd = sd
         self.skip_contents = skip_contents
         self.overwrite_saves = overwrite_saves
@@ -162,25 +220,67 @@ class CustomInstall:
     def copy_with_progress(self, src: BinaryIO, dst: BinaryIO, size: int, path: str, fire_event: bool = True):
         left = size
         cipher = self.crypto.create_ctr_cipher(Keyslot.SD, self.crypto.sd_path_to_iv(path))
+        hasher = sha256()
         while left > 0:
             to_read = min(READ_SIZE, left)
-            data = cipher.encrypt(src.read(READ_SIZE))
-            dst.write(data)
+            data = src.read(READ_SIZE)
+            hasher.update(data)
+            dst.write(cipher.encrypt(data))
             left -= to_read
             total_read = size - left
             if fire_event:
                 self.event.update_percentage((total_read / size) * 100, total_read / 1048576, size / 1048576)
-    
-    def start(self, continue_on_fail=True):
+
+        return hasher.digest()
+
+    @staticmethod
+    def get_reader(path: 'Union[PathLike, bytes, str]'):
+        if isdir(path):
+            # try the default tmd file
+            reader = CDNReader(join(path, 'tmd'))
+        else:
+            try:
+                reader = CIAReader(path)
+            except CIAError:
+                # if there was an error with parsing the CIA header,
+                # the file would be tried in CDNReader next (assuming it's a tmd)
+                # any other error should be propagated to the caller
+                reader = CDNReader(path)
+        return reader
+
+    def prepare_titles(self, paths: 'List[PathLike]'):
+        readers = []
+        for path in paths:
+            self.log(f'Reading {path}')
+            try:
+                reader = self.get_reader(path)
+            except (CIAError, CDNError, TitleMetadataError):
+                self.log(f"Couldn't read {path}, likely corrupt or not a CIA or CDN title")
+                continue
+            if reader.tmd.title_id.startswith('00048'):  # DSiWare
+                self.log(f'Skipping {reader.tmd.title_id} - DSiWare is not supported')
+                continue
+            readers.append((reader, path))
+        self.readers = readers
+
+    def check_size(self):
+        total_size = 0
+        for r, _ in self.readers:
+            total_size += get_install_size(r)
+
+        free_space = get_free_space(self.sd)
+        return total_size, free_space
+
+    def start(self):
         if frozen:
             save3ds_fuse_path = join(script_dir, 'bin', 'save3ds_fuse')
         else:
-            save3ds_fuse_path = join(script_dir, 'bin', sys.platform, 'save3ds_fuse')
+            save3ds_fuse_path = join(script_dir, 'bin', platform, 'save3ds_fuse')
         if is_windows:
             save3ds_fuse_path += '.exe'
         if not isfile(save3ds_fuse_path):
             self.log("Couldn't find " + save3ds_fuse_path, 2)
-            return None, False
+            return None, False, 0
 
         crypto = self.crypto
         # TODO: Move a lot of these into their own methods
@@ -191,293 +291,358 @@ class CustomInstall:
                               f'please remove extra directories')
         elif len(id1s) == 0:
             raise SDPathError(f'Could not find a suitable id1 directory for id0 {crypto.id0.hex()}')
+        id1 = id1s[0]
+        sd_path = join(sd_path, id1)
 
         if self.cifinish_out:
             cifinish_path = self.cifinish_out
         else:
             cifinish_path = join(self.sd, 'cifinish.bin')
-        sd_path = join(sd_path, id1s[0])
-        title_info_entries = {}
-        cifinish_data = load_cifinish(cifinish_path)
 
-        load_seeddb(self.seeddb)
+        try:
+            cifinish_data = load_cifinish(cifinish_path)
+        except InvalidCIFinishError as e:
+            self.log(f'{type(e).__qualname__}: {e}')
+            self.log(f'{cifinish_path} was corrupt!\n'
+                     f'This could mean an issue with the SD card or the filesystem. Please check it for errors.\n'
+                     f'It is also possible, though less likely, to be an issue with custom-install.\n'
+                     f'Exiting now to prevent possible issues. If you want to try again, delete cifinish.bin from the SD card and re-run custom-install.')
+            return None, False, 0
 
-        # Now loop through all provided cia files
-        
-        for idx, c in enumerate(self.cias):
-            self.log('Reading ' + c)
+        db_path = join(sd_path, 'dbs')
+        titledb_path = join(db_path, 'title.db')
+        importdb_path = join(db_path, 'import.db')
+        if not isfile(titledb_path):
+            makedirs(db_path, exist_ok=True)
+            with gzip.open(join(script_dir, 'title.db.gz')) as f:
+                tdb = f.read()
 
-            try:
-                cia = CIAReader(c)
-            except Exception as e:
-                self.event.on_error(sys.exc_info())
-                if continue_on_fail:
-                    continue
-                else:
-                    return None, False
+            self.log(f'Creating title.db...')
+            with open(titledb_path, 'wb') as o:
+                with self.crypto.create_ctr_io(Keyslot.SD, o, self.crypto.sd_path_to_iv('/dbs/title.db')) as e:
+                    e.write(tdb)
 
-            self.event.on_cia_start(idx)
+                    cmac = crypto.create_cmac_object(Keyslot.CMACSDNAND)
+                    cmac_data = [b'CTR-9DB0', 0x2.to_bytes(4, 'little'), tdb[0x100:0x200]]
+                    cmac.update(sha256(b''.join(cmac_data)).digest())
 
-            self.cia = cia
-            
-            tid_parts = (cia.tmd.title_id[0:8], cia.tmd.title_id[8:16])
+                    e.seek(0)
+                    e.write(cmac.digest())
 
-            try:
-                self.log(f'Installing {cia.contents[0].exefs.icon.get_app_title().short_desc}...')
-            except:
-                self.log('Installing...')
-            
-            sizes = [1] * 5
+            self.log(f'Creating import.db...')
+            with open(importdb_path, 'wb') as o:
+                with self.crypto.create_ctr_io(Keyslot.SD, o, self.crypto.sd_path_to_iv('/dbs/import.db')) as e:
+                    e.write(tdb)
 
-            if cia.tmd.save_size:
-                # one for the data directory, one for the 00000001.sav file
-                sizes.extend((1, cia.tmd.save_size))
-            
-            for record in cia.content_info:
-                sizes.append(record.size)
-            
-            # this calculates the size to put in the Title Info Entry
-            title_size = sum(roundup(x, TITLE_ALIGN_SIZE) for x in sizes)
+                    cmac = crypto.create_cmac_object(Keyslot.CMACSDNAND)
+                    cmac_data = [b'CTR-9DB0', 0x3.to_bytes(4, 'little'), tdb[0x100:0x200]]
+                    cmac.update(sha256(b''.join(cmac_data)).digest())
 
-            # checks if this is dlc, which has some differences
-            is_dlc = tid_parts[0] == '0004008c'
+                    e.seek(0)
+                    e.write(cmac.digest())
 
-            # this checks if it has a manual (index 1) and is not DLC
-            has_manual = (not is_dlc) and (1 in cia.contents)
+            del tdb
 
-            # this gets the extdata id from the extheader, stored in the storage info area
-            try:
-                with cia.contents[0].open_raw_section(NCCHSection.ExtendedHeader) as e:
-                    e.seek(0x200 + 0x30)
-                    extdata_id = e.read(8)
-            except KeyError:
-                # not an executable title
-                extdata_id = b'\0' * 8
-
-            # cmd content id, starts with 1 for non-dlc contents
-            cmd_id = len(cia.content_info) if is_dlc else 1
-            cmd_filename = f'{cmd_id:08x}.cmd'
-
-            # get the title root where all the contents will be
-            title_root = join(sd_path, 'title', *tid_parts)
-            content_root = join(title_root, 'content')
-            # generate the path used for the IV
-            title_root_cmd = f'/title/{"/".join(tid_parts)}'
-            content_root_cmd = title_root_cmd + '/content'
-
-            if not self.skip_contents:
-                makedirs(join(content_root, 'cmd'), exist_ok=True)
-                if cia.tmd.save_size:
-                    makedirs(join(title_root, 'data'), exist_ok=True)
-                if is_dlc:
-                    # create the separate directories for every 256 contents
-                    for x in range(((len(cia.content_info) - 1) // 256) + 1):
-                        makedirs(join(content_root, f'{x:08x}'), exist_ok=True)
-
-                # maybe this will be changed in the future
-                tmd_id = 0
-
-                tmd_filename = f'{tmd_id:08x}.tmd'
-
-                # write the tmd
-                enc_path = content_root_cmd + '/' + tmd_filename
-                self.log(f'Writing {enc_path}...')
-                with cia.open_raw_section(CIASection.TitleMetadata) as s:
-                    with open(join(content_root, tmd_filename), 'wb') as o:
-                        self.copy_with_progress(s, o, cia.sections[CIASection.TitleMetadata].size, enc_path,
-                                                fire_event=False)
-
-                # write each content
-                for co in cia.content_info:
-                    content_filename = co.id + '.app'
-                    if is_dlc:
-                        dir_index = format((co.cindex // 256), '08x')
-                        enc_path = content_root_cmd + f'/{dir_index}/{content_filename}'
-                        out_path = join(content_root, dir_index, content_filename)
-                    else:
-                        enc_path = content_root_cmd + '/' + content_filename
-                        out_path = join(content_root, content_filename)
-                    self.log(f'Writing {enc_path}...')
-                    with cia.open_raw_section(co.cindex) as s, open(out_path, 'wb') as o:
-                        self.copy_with_progress(s, o, co.size, enc_path)
-
-                # generate a blank save
-                if cia.tmd.save_size:
-                    enc_path = title_root_cmd + '/data/00000001.sav'
-                    out_path = join(title_root, 'data', '00000001.sav')
-                    if self.overwrite_saves or not isfile(out_path):
-                        cipher = crypto.create_ctr_cipher(Keyslot.SD, crypto.sd_path_to_iv(enc_path))
-                        # in a new save, the first 0x20 are all 00s. the rest can be random
-                        data = cipher.encrypt(b'\0' * 0x20)
-                        self.log(f'Generating blank save at {enc_path}...')
-                        with open(out_path, 'wb') as o:
-                            o.write(data)
-                            o.write(b'\0' * (cia.tmd.save_size - 0x20))
-                    else:
-                        self.log(f'Not overwriting existing save at {enc_path}')
-
-                # generate and write cmd
-                enc_path = content_root_cmd + '/cmd/' + cmd_filename
-                out_path = join(content_root, 'cmd', cmd_filename)
-                self.log(f'Generating {enc_path}')
-                highest_index = 0
-                content_ids = {}
-
-                for record in cia.content_info:
-                    highest_index = record.cindex
-                    with cia.open_raw_section(record.cindex) as s:
-                        s.seek(0x100)
-                        cmac_data = s.read(0x100)
-
-                    id_bytes = bytes.fromhex(record.id)[::-1]
-                    cmac_data += record.cindex.to_bytes(4, 'little') + id_bytes
-
-                    cmac_ncch = crypto.create_cmac_object(Keyslot.CMACSDNAND)
-                    cmac_ncch.update(sha256(cmac_data).digest())
-                    content_ids[record.cindex] = (id_bytes, cmac_ncch.digest())
-
-                # add content IDs up to the last one
-                ids_by_index = [CMD_MISSING] * (highest_index + 1)
-                installed_ids = []
-                cmacs = []
-                for x in range(len(ids_by_index)):
-                    try:
-                        info = content_ids[x]
-                    except KeyError:
-                        # "MISSING CONTENT!"
-                        # The 3DS does generate a cmac for missing contents, but I don't know how it works.
-                        # It doesn't matter anyway, the title seems to be fully functional.
-                        cmacs.append(bytes.fromhex('4D495353494E4720434F4E54454E5421'))
-                    else:
-                        ids_by_index[x] = info[0]
-                        cmacs.append(info[1])
-                        installed_ids.append(info[0])
-                installed_ids.sort(key=lambda x: int.from_bytes(x, 'little'))
-
-                final = (cmd_id.to_bytes(4, 'little')
-                         + len(ids_by_index).to_bytes(4, 'little')
-                         + len(installed_ids).to_bytes(4, 'little')
-                         + (1).to_bytes(4, 'little'))
-                cmac_cmd_header = crypto.create_cmac_object(Keyslot.CMACSDNAND)
-                cmac_cmd_header.update(final)
-                final += cmac_cmd_header.digest()
-
-                final += b''.join(ids_by_index)
-                final += b''.join(installed_ids)
-                final += b''.join(cmacs)
-
-                cipher = crypto.create_ctr_cipher(Keyslot.SD, crypto.sd_path_to_iv(enc_path))
-                self.log(f'Writing {enc_path}')
-                with open(out_path, 'wb') as o:
-                    o.write(cipher.encrypt(final))
-
-            # this starts building the title info entry
-            title_info_entry_data = [
-                # title size
-                title_size.to_bytes(8, 'little'),
-                # title type, seems to usually be 0x40
-                0x40.to_bytes(4, 'little'),
-                # title version
-                int(cia.tmd.title_version).to_bytes(2, 'little'),
-                # ncch version
-                cia.contents[0].version.to_bytes(2, 'little'),
-                # flags_0, only checking if there is a manual
-                (1 if has_manual else 0).to_bytes(4, 'little'),
-                # tmd content id, always starting with 0
-                (0).to_bytes(4, 'little'),
-                # cmd content id
-                cmd_id.to_bytes(4, 'little'),
-                # flags_1, only checking save data
-                (1 if cia.tmd.save_size else 0).to_bytes(4, 'little'),
-                # extdataid low
-                extdata_id[0:4],
-                # reserved
-                b'\0' * 4,
-                # flags_2, only using a common value
-                0x100000000.to_bytes(8, 'little'),
-                # product code
-                cia.contents[0].product_code.encode('ascii').ljust(0x10, b'\0'),
-                # reserved
-                b'\0' * 0x10,
-                # unknown
-                randint(0, 0xFFFFFFFF).to_bytes(4, 'little'),
-                # reserved
-                b'\0' * 0x2c
+        with TemporaryDirectory(suffix='-custom-install') as tempdir:
+            # set up the common arguments for the two times we call save3ds_fuse
+            save3ds_fuse_common_args = [
+                save3ds_fuse_path,
+                '-b', crypto.b9_path,
+                '-m', self.movable,
+                '--sd', self.sd,
+                '--db', 'sdtitle',
+                tempdir
             ]
 
-            title_info_entries[cia.tmd.title_id] = b''.join(title_info_entry_data)
+            extra_kwargs = {}
+            if is_windows:
+                # hide console window
+                extra_kwargs['creationflags'] = 0x08000000  # CREATE_NO_WINDOW
 
-            cifinish_data[int(cia.tmd.title_id, 16)] = {'seed': (get_seed(cia.contents[0].program_id) if cia.contents[0].flags.uses_seed else None)}
+            # extract the title database to add our own entry to
+            self.log('Extracting Title Database...')
+            out = subprocess.run(save3ds_fuse_common_args + ['-x'],
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT,
+                                 encoding='utf-8',
+                                 **extra_kwargs)
+            if out.returncode:
+                for l in out.stdout.split('\n'):
+                    self.log(l)
+                self.log('Command line:')
+                for l in pformat(out.args).split('\n'):
+                    self.log(l)
+                return None, False, 0
 
-        # This is saved regardless if any titles were installed, so the file can be upgraded just in case.
-        save_cifinish(cifinish_path, cifinish_data)
+            if self.seeddb:
+                load_seeddb(self.seeddb)
 
-        if title_info_entries:
-            with TemporaryDirectory(suffix='-custom-install') as tempdir:
-                # set up the common arguments for the two times we call save3ds_fuse
-                save3ds_fuse_common_args = [
-                    save3ds_fuse_path,
-                    '-b', crypto.b9_path,
-                    '-m', self.movable,
-                    '--sd', self.sd,
-                    '--db', 'sdtitle',
-                    tempdir
+            install_state = {'installed': [], 'failed': []}
+
+            # Now loop through all provided cia files
+            for idx, info in enumerate(self.readers):
+                cia, path = info
+
+                self.event.on_cia_start(idx)
+                self.event.update_status(path, InstallStatus.Starting)
+
+                temp_title_root = join(self.sd, f'ci-install-temp-{cia.tmd.title_id}-{randint(0, 0xFFFFFFFF):08x}')
+                makedirs(temp_title_root, exist_ok=True)
+
+                tid_parts = (cia.tmd.title_id[0:8], cia.tmd.title_id[8:16])
+
+                try:
+                    display_title = f'{cia.contents[0].exefs.icon.get_app_title().short_desc} - {cia.tmd.title_id}'
+                except:
+                    display_title = cia.tmd.title_id
+                self.log(f'Installing {display_title}...')
+
+                title_size = get_install_size(cia)
+
+                # checks if this is dlc, which has some differences
+                is_dlc = tid_parts[0] == '0004008c'
+
+                # this checks if it has a manual (index 1) and is not DLC
+                has_manual = (not is_dlc) and (1 in cia.contents)
+
+                # this gets the extdata id from the extheader, stored in the storage info area
+                try:
+                    with cia.contents[0].open_raw_section(NCCHSection.ExtendedHeader) as e:
+                        e.seek(0x200 + 0x30)
+                        extdata_id = e.read(8)
+                except KeyError:
+                    # not an executable title
+                    extdata_id = b'\0' * 8
+
+                # cmd content id, starts with 1 for non-dlc contents
+                cmd_id = len(cia.content_info) if is_dlc else 1
+                cmd_filename = f'{cmd_id:08x}.cmd'
+
+                # this is where the final directory will be moved
+                tidhigh_root = join(sd_path, 'title', tid_parts[0])
+
+                # get the title root where all the contents will be
+                title_root = join(sd_path, 'title', *tid_parts)
+                content_root = join(title_root, 'content')
+                # generate the path used for the IV
+                title_root_cmd = f'/title/{"/".join(tid_parts)}'
+                content_root_cmd = title_root_cmd + '/content'
+
+                temp_content_root = join(temp_title_root, 'content')
+
+                if not self.skip_contents:
+                    self.event.update_status(path, InstallStatus.Writing)
+                    makedirs(join(temp_content_root, 'cmd'), exist_ok=True)
+                    if cia.tmd.save_size:
+                        makedirs(join(temp_title_root, 'data'), exist_ok=True)
+                    if is_dlc:
+                        # create the separate directories for every 256 contents
+                        for x in range(((len(cia.content_info) - 1) // 256) + 1):
+                            makedirs(join(temp_content_root, f'{x:08x}'), exist_ok=True)
+
+                    # maybe this will be changed in the future
+                    tmd_id = 0
+
+                    tmd_filename = f'{tmd_id:08x}.tmd'
+
+                    # write the tmd
+                    tmd_enc_path = content_root_cmd + '/' + tmd_filename
+                    self.log(f'Writing {tmd_enc_path}...')
+                    with open(join(temp_content_root, tmd_filename), 'wb') as o:
+                        with self.crypto.create_ctr_io(Keyslot.SD, o, self.crypto.sd_path_to_iv(tmd_enc_path)) as e:
+                            e.write(bytes(cia.tmd))
+
+                    # in case the contents are corrupted
+                    do_continue = False
+                    # write each content
+                    for co in cia.content_info:
+                        content_filename = co.id + '.app'
+                        if is_dlc:
+                            dir_index = format((co.cindex // 256), '08x')
+                            content_enc_path = content_root_cmd + f'/{dir_index}/{content_filename}'
+                            content_out_path = join(temp_content_root, dir_index, content_filename)
+                        else:
+                            content_enc_path = content_root_cmd + '/' + content_filename
+                            content_out_path = join(temp_content_root, content_filename)
+                        self.log(f'Writing {content_enc_path}...')
+                        with cia.open_raw_section(co.cindex) as s, open(content_out_path, 'wb') as o:
+                            result_hash = self.copy_with_progress(s, o, co.size, content_enc_path)
+                            if result_hash != co.hash:
+                                self.log(f'WARNING: Hash does not match for {content_enc_path}!')
+                                install_state['failed'].append(display_title)
+                                rename(temp_title_root, temp_title_root + '-corrupted')
+                                do_continue = True
+                                self.event.update_status(path, InstallStatus.Failed)
+                                break
+
+                    if do_continue:
+                        continue
+
+                    # generate a blank save
+                    if cia.tmd.save_size:
+                        sav_enc_path = title_root_cmd + '/data/00000001.sav'
+                        tmp_sav_out_path = join(temp_title_root, 'data', '00000001.sav')
+                        sav_out_path = join(title_root, 'data', '00000001.sav')
+                        if self.overwrite_saves or not isfile(sav_out_path):
+                            cipher = crypto.create_ctr_cipher(Keyslot.SD, crypto.sd_path_to_iv(sav_enc_path))
+                            # in a new save, the first 0x20 are all 00s. the rest can be random
+                            data = cipher.encrypt(b'\0' * 0x20)
+                            self.log(f'Generating blank save at {sav_enc_path}...')
+                            with open(tmp_sav_out_path, 'wb') as o:
+                                o.write(data)
+                                o.write(b'\0' * (cia.tmd.save_size - 0x20))
+                        else:
+                            self.log(f'Copying original save file from {sav_enc_path}...')
+                            copy2(sav_out_path, tmp_sav_out_path)
+
+                    # generate and write cmd
+                    cmd_enc_path = content_root_cmd + '/cmd/' + cmd_filename
+                    cmd_out_path = join(temp_content_root, 'cmd', cmd_filename)
+                    self.log(f'Generating {cmd_enc_path}')
+                    highest_index = 0
+                    content_ids = {}
+
+                    for record in cia.content_info:
+                        highest_index = record.cindex
+                        with cia.open_raw_section(record.cindex) as s:
+                            s.seek(0x100)
+                            cmac_data = s.read(0x100)
+
+                        id_bytes = bytes.fromhex(record.id)[::-1]
+                        cmac_data += record.cindex.to_bytes(4, 'little') + id_bytes
+
+                        cmac_ncch = crypto.create_cmac_object(Keyslot.CMACSDNAND)
+                        cmac_ncch.update(sha256(cmac_data).digest())
+                        content_ids[record.cindex] = (id_bytes, cmac_ncch.digest())
+
+                    # add content IDs up to the last one
+                    ids_by_index = [CMD_MISSING] * (highest_index + 1)
+                    installed_ids = []
+                    cmacs = []
+                    for x in range(len(ids_by_index)):
+                        try:
+                            info = content_ids[x]
+                        except KeyError:
+                            # "MISSING CONTENT!"
+                            # The 3DS does generate a cmac for missing contents, but I don't know how it works.
+                            # It doesn't matter anyway, the title seems to be fully functional.
+                            cmacs.append(bytes.fromhex('4D495353494E4720434F4E54454E5421'))
+                        else:
+                            ids_by_index[x] = info[0]
+                            cmacs.append(info[1])
+                            installed_ids.append(info[0])
+                    installed_ids.sort(key=lambda x: int.from_bytes(x, 'little'))
+
+                    final = (cmd_id.to_bytes(4, 'little')
+                             + len(ids_by_index).to_bytes(4, 'little')
+                             + len(installed_ids).to_bytes(4, 'little')
+                             + (1).to_bytes(4, 'little'))
+                    cmac_cmd_header = crypto.create_cmac_object(Keyslot.CMACSDNAND)
+                    cmac_cmd_header.update(final)
+                    final += cmac_cmd_header.digest()
+
+                    final += b''.join(ids_by_index)
+                    final += b''.join(installed_ids)
+                    final += b''.join(cmacs)
+
+                    cipher = crypto.create_ctr_cipher(Keyslot.SD, crypto.sd_path_to_iv(cmd_enc_path))
+                    self.log(f'Writing {cmd_enc_path}')
+                    with open(cmd_out_path, 'wb') as o:
+                        o.write(cipher.encrypt(final))
+
+                # this starts building the title info entry
+                title_info_entry_data = [
+                    # title size
+                    title_size.to_bytes(8, 'little'),
+                    # title type, seems to usually be 0x40
+                    0x40.to_bytes(4, 'little'),
+                    # title version
+                    int(cia.tmd.title_version).to_bytes(2, 'little'),
+                    # ncch version
+                    cia.contents[0].version.to_bytes(2, 'little'),
+                    # flags_0, only checking if there is a manual
+                    (1 if has_manual else 0).to_bytes(4, 'little'),
+                    # tmd content id, always starting with 0
+                    (0).to_bytes(4, 'little'),
+                    # cmd content id
+                    cmd_id.to_bytes(4, 'little'),
+                    # flags_1, only checking save data
+                    (1 if cia.tmd.save_size else 0).to_bytes(4, 'little'),
+                    # extdataid low
+                    extdata_id[0:4],
+                    # reserved
+                    b'\0' * 4,
+                    # flags_2, only using a common value
+                    0x100000000.to_bytes(8, 'little'),
+                    # product code
+                    cia.contents[0].product_code.encode('ascii').ljust(0x10, b'\0'),
+                    # reserved
+                    b'\0' * 0x10,
+                    # unknown
+                    randint(0, 0xFFFFFFFF).to_bytes(4, 'little'),
+                    # reserved
+                    b'\0' * 0x2c
                 ]
 
-                extra_kwargs = {}
-                if is_windows:
-                    # hide console window
-                    extra_kwargs['creationflags'] = 0x08000000  # CREATE_NO_WINDOW
+                self.event.update_status(path, InstallStatus.Finishing)
+                if isdir(title_root):
+                    self.log(f'Removing original install at {title_root}...')
+                    rmtree(title_root)
 
-                # extract the title database to add our own entry to
-                self.log('Extracting Title Database...')
-                out = subprocess.run(save3ds_fuse_common_args + ['-x'],
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.STDOUT,
-                                     encoding=getpreferredencoding(),
-                                     **extra_kwargs)
-                if out.returncode:
-                    for l in out.stdout.split('\n'):
-                        self.log(l)
-                    return False
+                makedirs(tidhigh_root, exist_ok=True)
+                rename(temp_title_root, title_root)
 
-                for title_id, entry in title_info_entries.items():
-                    # write the title info entry to the temp directory
-                    with open(join(tempdir, title_id), 'wb') as o:
-                        o.write(entry)
+                cifinish_data[int(cia.tmd.title_id, 16)] = {'seed': (get_seed(cia.contents[0].program_id) if cia.contents[0].flags.uses_seed else None)}
+
+                # This is saved regardless if any titles were installed, so the file can be upgraded just in case.
+                save_cifinish(cifinish_path, cifinish_data)
+
+                with open(join(tempdir, cia.tmd.title_id), 'wb') as o:
+                    o.write(b''.join(title_info_entry_data))
 
                 # import the directory, now including our title
                 self.log('Importing into Title Database...')
                 out = subprocess.run(save3ds_fuse_common_args + ['-i'],
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.STDOUT,
-                                     encoding=getpreferredencoding(),
+                                     encoding='utf-8',
                                      **extra_kwargs)
                 if out.returncode:
                     for l in out.stdout.split('\n'):
                         self.log(l)
-                    return False, False
+                    self.log('Command line:')
+                    for l in pformat(out.args).split('\n'):
+                        self.log(l)
+                    install_state['failed'].append(display_title)
+                    self.event.update_status(path, InstallStatus.Failed)
+                else:
+                    install_state['installed'].append(display_title)
+                    self.event.update_status(path, InstallStatus.Done)
 
-            finalize_3dsx_orig_path = join(script_dir, 'custom-install-finalize.3dsx')
-            hb_dir = join(self.sd, '3ds')
-            finalize_3dsx_path = join(hb_dir, 'custom-install-finalize.3dsx')
             copied = False
-            if isfile(finalize_3dsx_orig_path):
-                self.log('Copying finalize program to ' + finalize_3dsx_path)
-                makedirs(hb_dir, exist_ok=True)
-                copyfile(finalize_3dsx_orig_path, finalize_3dsx_path)
-                copied = True
+            # launchable applications, not DLC or update data
+            application_count = len(glob(join(tempdir, '00040000*')))
+            if install_state['installed']:
+                if application_count >= 300:
+                    self.log(f'{application_count} installed applications were detected.', 1)
+                    self.log('The HOME Menu will only show 300 icons.', 1)
+                    self.log('Some applications (not updates or DLC) will need to be deleted.', 1)
+                finalize_3dsx_orig_path = join(script_dir, 'custom-install-finalize.3dsx')
+                hb_dir = join(self.sd, '3ds')
+                finalize_3dsx_path = join(hb_dir, 'custom-install-finalize.3dsx')
+                if isfile(finalize_3dsx_orig_path):
+                    self.log('Copying finalize program to ' + finalize_3dsx_path)
+                    makedirs(hb_dir, exist_ok=True)
+                    copyfile(finalize_3dsx_orig_path, finalize_3dsx_path)
+                    copied = True
 
-            self.log('FINAL STEP:')
-            self.log('Run custom-install-finalize through homebrew launcher.')
-            self.log('This will install a ticket and seed if required.')
-            if copied:
-                self.log('custom-install-finalize has been copied to the SD card.')
-            return True, copied
+                self.log('FINAL STEP:')
+                self.log('Run custom-install-finalize through homebrew launcher.')
+                self.log('This will install a ticket and seed if required.')
+                if copied:
+                    self.log('custom-install-finalize has been copied to the SD card.')
 
-        else:
-            self.log('Did not install any titles.', 2)
-            return None, False
+            return install_state, copied, application_count
 
     def get_sd_path(self):
         sd_path = join(self.sd, 'Nintendo 3DS', self.crypto.id0.hex())
@@ -522,7 +687,7 @@ class CustomInstall:
 
 
 if __name__ == "__main__":
-    parser = ArgumentParser(description='Manually install a CIA to the SD card for a Nintendo 3DS system.')
+    parser = ArgumentParser(description='Install a CIA to the SD card for a Nintendo 3DS system.')
     parser.add_argument('cia', help='CIA files', nargs='+')
     parser.add_argument('-m', '--movable', help='movable.sed file', required=True)
     parser.add_argument('-b', '--boot9', help='boot9 file')
@@ -532,11 +697,11 @@ if __name__ == "__main__":
     parser.add_argument('--overwrite-saves', help='overwrite existing save files', action='store_true')
     parser.add_argument('--cifinish-out', help='path for cifinish.bin file, defaults to (SD root)/cifinish.bin')
 
+    print(f'custom-install {CI_VERSION} - https://github.com/ihaveamac/custom-install')
     args = parser.parse_args()
 
     installer = CustomInstall(boot9=args.boot9,
                               seeddb=args.seeddb,
-                              cias=args.cia,
                               movable=args.movable,
                               sd=args.sd,
                               overwrite_saves=args.overwrite_saves,
@@ -558,7 +723,21 @@ if __name__ == "__main__":
     installer.event.update_percentage += percent_handle
     installer.event.on_error += error
 
-    result, copied_3dsx = installer.start(continue_on_fail=False)
+    installer.prepare_titles(args.cia)
+
+    if not args.skip_contents:
+        total_size, free_space = installer.check_size()
+        if total_size > free_space:
+            installer.event.on_log_msg(f'Not enough free space.\n'
+                                       f'Combined title install size: {total_size / (1024 * 1024):0.2f} MiB\n'
+                                       f'Free space: {free_space / (1024 * 1024):0.2f} MiB')
+            sys.exit(1)
+
+    result, copied_3dsx, application_count = installer.start()
     if result is False:
         # save3ds_fuse failed
         installer.log('NOTE: Once save3ds_fuse is fixed, run the same command again with --skip-contents')
+    if application_count >= 300:
+        installer.log(f'\n\nWarning: {application_count} installed applications were detected.\n'
+                      f'The HOME Menu will only show 300 icons.\n'
+                      f'Some applications (not updates or DLC) will need to be deleted.')

finalize/source/main.c

@@ -189,6 +189,7 @@ int load_cifinish(char* path, struct finish_db_entry_final **entries)
 		}
 	}
 
+	fclose(fp);
 	return header.title_count;
 
 fail:
@@ -196,34 +197,99 @@ fail:
 	return -1;
 }
 
+Result check_title_exist(u64 title_id, u64 *ticket_ids, u32 ticket_ids_length,  u64 *title_ids, u32 title_ids_length)
+{
+	Result ret = -2;
+
+	for (u32 i = 0; i < ticket_ids_length; i++)
+	{
+		if (ticket_ids[i] == title_id)
+		{
+			ret++;
+			break;
+		}
+	}
+
+	for (u32 i = 0; i < title_ids_length; i++)
+	{
+		if (title_ids[i] == title_id)
+		{
+			ret++;
+			break;
+		}
+	}
+
+	return ret;
+}
+
 void finalize_install(void)
 {
 	Result res;
 	Handle ticketHandle;
 	struct ticket_dumb ticket_buf;
-	struct finish_db_entry_final *entries;
-	int title_count;
+	struct finish_db_entry_final *entries = NULL;
+	int title_count;	
+
+	u32 titles_read;
+	u32 tickets_read;
+
+	res = AM_GetTitleCount(MEDIATYPE_SD, &titles_read);
+
+	if (R_FAILED(res))
+	{
+		return;
+	}
+
+	res = AM_GetTicketCount(&tickets_read);
+
+	if (R_FAILED(res))
+	{
+		return;
+	}
+
+	u64 *installed_ticket_ids = malloc(sizeof(u64) * tickets_read );
+	u64 *installed_title_ids  = malloc(sizeof(u64) * titles_read  );
+
+	res = AM_GetTitleList(&titles_read, MEDIATYPE_SD, titles_read, installed_title_ids);
+
+	if (R_FAILED(res))
+	{
+		goto exit;
+	}
+
+	res = AM_GetTicketList(&tickets_read, tickets_read, 0, installed_ticket_ids);
+
+	if (R_FAILED(res))
+	{
+		goto exit;
+	}
 
 	title_count = load_cifinish(CIFINISH_PATH, &entries);
+
 	if (title_count == -1)
 	{
-		free(entries);
-		return;
+		goto exit;
 	}
-	if (title_count == 0)
+	else if (title_count == 0)
 	{
 		printf("No titles to finalize.\n");
-		free(entries);
-		return;
+		goto exit;
 	}
 
-	//printf("Deleting %s...\n", CIFINISH_PATH);
-	//unlink(CIFINISH_PATH);
-
 	memcpy(&ticket_buf, basetik_bin, basetik_bin_size);
 
+	Result exist_res = 0;
+
 	for (int i = 0; i < title_count; ++i)
 	{
+		exist_res = check_title_exist(entries[i].title_id, installed_ticket_ids, tickets_read, installed_title_ids, titles_read);
+
+		if (R_SUCCEEDED(exist_res))
+		{
+			printf("No need to finalize %016llx, skipping...\n", entries[i].title_id);
+			continue;
+		}
+
 		printf("Finalizing %016llx...\n", entries[i].title_id);
 
 		ticket_buf.title_id_be = __builtin_bswap64(entries[i].title_id);
@@ -233,8 +299,7 @@ void finalize_install(void)
 		{
 			printf("Failed to begin ticket install: %08lx\n", res);
 			AM_InstallTicketAbort(ticketHandle);
-			free(entries);
-			return;
+			goto exit;
 		}
 
 		res = FSFILE_Write(ticketHandle, NULL, 0, &ticket_buf, sizeof(struct ticket_dumb), 0);
@@ -242,8 +307,7 @@ void finalize_install(void)
 		{
 			printf("Failed to write ticket: %08lx\n", res);
 			AM_InstallTicketAbort(ticketHandle);
-			free(entries);
-			return;
+			goto exit;
 		}
 
 		res = AM_InstallTicketFinish(ticketHandle);
@@ -251,8 +315,7 @@ void finalize_install(void)
 		{
 			printf("Failed to finish ticket install: %08lx\n", res);
 			AM_InstallTicketAbort(ticketHandle);
-			free(entries);
-			return;
+			goto exit;
 		}
 
 		if (entries[i].has_seed)
@@ -266,7 +329,15 @@ void finalize_install(void)
 		}
 	}
 
+	printf("Deleting %s...\n", CIFINISH_PATH);
+	unlink(CIFINISH_PATH);
+
+	exit:
+
 	free(entries);
+	free(installed_ticket_ids);
+	free(installed_title_ids);
+	return;
 }
 
 int main(int argc, char* argv[])
@@ -275,7 +346,7 @@ int main(int argc, char* argv[])
 	gfxInitDefault();
 	consoleInit(GFX_TOP, NULL);
 
-	printf("custom-install-finalize v1.4\n");
+	printf("custom-install-finalize v1.6\n");
 
 	finalize_install();
 	// print this at the end in case it gets pushed off the screen

make-standalone.bat

@@ -6,6 +6,7 @@ copy TaskbarLib.tlb build\custom-install-standalone
 copy bin\win32\save3ds_fuse.exe build\custom-install-standalone\bin
 copy bin\README build\custom-install-standalone\bin
 copy custom-install-finalize.3dsx build\custom-install-standalone
+copy title.db.gz build\custom-install-standalone
 copy extras\windows-quickstart.txt build\custom-install-standalone
 copy LICENSE.md build\custom-install-standalone
 python -m zipfile -c dist\custom-install-standalone.zip build\custom-install-standalone

requirements-win32.txt

@@ -1,2 +1,2 @@
 -r requirements.txt
-comtypes==1.1.7
+comtypes==1.1.10

requirements.txt

@@ -1,3 +1,2 @@
-pycryptodomex==3.9.8
-events==0.3
-pyctr==0.4.1
+events==0.4
+pyctr>=0.4,<0.6