Cache the changed build files instead of the public folder

agent/build_configs/get_cached_app.py

@@ -109,41 +109,64 @@ def link_node_modules(bench_path: str, app_name: str, app_public_path: str) -> N
     if os.path.isdir(node_modules) and not os.path.lexists(link):
         os.symlink(node_modules, link)
 
-
 def extract_assets(app_name: str, tar_path: str) -> None:
-    """Replace the app's public folder with the stored one, and restore the links bench build makes."""
+    """Apply the stored changed build files over the app, and restore the links bench build makes."""
     if not hasattr(tarfile, "data_filter"):
         raise RuntimeError("this Python can't extract archives safely")
 
     change_working_directory()
     bench_path = os.getcwd()
-    app_public_path = os.path.join(bench_path, "apps", app_name, app_name, "public")
+    app_path = os.path.join(bench_path, "apps", app_name)
+    app_public_path = os.path.join(app_path, app_name, "public")
     assets_path = os.path.join(bench_path, "sites", "assets", app_name)
 
     with tempfile.TemporaryDirectory(dir=os.path.join(bench_path, "sites")) as extract_path:
-        # The data filter refuses paths and links that point outside extract_path
+        # Extract to a scratch dir first so a bad archive can't leave the app half-updated.
+        # The data filter refuses paths and links that point outside extract_path.
         with tarfile.open(tar_path, mode="r:gz") as tar:
             tar.extractall(path=extract_path, filter="data")
 
-        extracted = os.path.join(extract_path, app_name)
-        if not os.path.isdir(extracted):
-            raise RuntimeError(f"stored assets have no {app_name} folder")
-
-        if os.path.islink(assets_path):
-            os.unlink(assets_path)
-        elif os.path.exists(assets_path):
-            shutil.rmtree(assets_path)
-        if os.path.exists(app_public_path):
-            shutil.rmtree(app_public_path)
-        shutil.move(extracted, app_public_path)
+        _overlay(extract_path, app_path)
 
     # Restore symlink since we deploy with cp -LR
+    if os.path.islink(assets_path):
+        os.unlink(assets_path)
+    elif os.path.exists(assets_path):
+        shutil.rmtree(assets_path)
     os.symlink(app_public_path, assets_path)
     link_node_modules(bench_path, app_name, app_public_path)
 
-    print(f"Assets moved to {app_public_path} and symlink restored at {assets_path}")
+    print(f"Build files applied to {app_path} and symlink restored at {assets_path}")
+
+
+def _overlay(src_root: str, dest_root: str) -> None:
+    """Move everything under src_root into dest_root, replacing what's there
+    entry by entry and leaving files that aren't in src_root alone."""
+    for root, dirs, files in os.walk(src_root):
+        dest_dir = os.path.normpath(os.path.join(dest_root, os.path.relpath(root, src_root)))
+
+        for name in list(dirs):
+            src, dest = os.path.join(root, name), os.path.join(dest_dir, name)
+            if os.path.islink(src):
+                # os.walk lists symlinks to dirs as dirs; move them as links.
+                _replace(src, dest)
+                dirs.remove(name)
+            else:
+                if os.path.islink(dest) or (os.path.lexists(dest) and not os.path.isdir(dest)):
+                    os.unlink(dest)
+                os.makedirs(dest, exist_ok=True)
+
+        for name in files:
+            _replace(os.path.join(root, name), os.path.join(dest_dir, name))
 
 
+def _replace(src: str, dest: str) -> None:
+    if os.path.islink(dest) or os.path.isfile(dest):
+        os.unlink(dest)
+    elif os.path.isdir(dest):
+        shutil.rmtree(dest)
+    shutil.move(src, dest)
+
 def main():
     """Use the app's stored assets if the agent put them in the build context, else build them"""
     env = os.environ.copy()
@@ -159,13 +182,38 @@ def main():
         try:
             extract_assets(APP_NAME, STORED_ASSETS)
             _update_assets_json(APP_NAME)
+            change_working_directory()
+            subprocess.run(["bench", "compile-po-to-mo", "--app", f"{APP_NAME}"], check=True, env=env)
+            subprocess.run(["../env/bin/python", "/usr/local/bin/run_hooks.py", f"{APP_NAME}"], check=True, env=env, cwd=os.getcwd()+"/sites")
             return
         except UNUSABLE_ARCHIVE_ERRORS as e:
             print(f"Stored assets could not be used: {e}")
 
     print(f"Assets {asset_filename} not found in store. Building...")
+    subprocess.run(["touch", f"/home/frappe/time_marker_{APP_NAME}"], check=True, env=env)
     build_assets(APP_NAME)
-
+    change_working_directory()
+    app_dir = os.getcwd()+f"/apps/{APP_NAME}"
+    list_path = f"/home/frappe/added_files_{APP_NAME}.txt"
+    tar_path = f"/home/frappe/added_files_{APP_NAME}.tar.gz"
+
+    with open(list_path, "w") as f:
+        subprocess.run(
+            [
+                "find", ".", "-mindepth", "1",
+                "-name", "node_modules", "-prune",
+                "-o",
+                "-cnewer", f"/home/frappe/time_marker_{APP_NAME}", "-printf", "%P\n",
+            ],
+            cwd=app_dir, stdout=f, check=True,
+        )
+
+    # --no-recursion: the list already names every changed file, and the
+    # directories in it should not pull in their unchanged contents.
+    subprocess.run(
+        ["tar", "-czf", tar_path, "--no-recursion", "-C", app_dir, "-T", list_path],
+        check=True,
+    )
 
 if __name__ == "__main__":
     main()

agent/build_configs/run_hooks.py

@@ -0,0 +1,24 @@
+import frappe
+import argparse
+
+parser = argparse.ArgumentParser(description="Process a single app name.")
+
+# Define a single positional argument (or use --app for an optional flag)
+parser.add_argument("app", type=str, help="Name of a single app")
+
+args = parser.parse_args()
+# Initialize Frappe with an empty site string (exactly how bench build does it)
+frappe.init("")
+apps = [args.app]
+try:
+    # Import your function
+    from frappe.commands.utils import run_after_build_hook
+    #from frappe.bundler import build_framework_islands
+
+    # enable once in release
+    #build_framework_islands(True)
+    # Pass the list of apps you want to simulate as built
+    run_after_build_hook(apps)
+
+finally:
+    frappe.destroy()

agent/builder.py

@@ -164,7 +164,7 @@ def _run_git_command(self, command: str, cwd: str) -> str:
 
     def _copy_build_config_files(self):
         """Copy generic build config files to the build context"""
-        for filename in ["common_site_config.json", "supervisord.conf", ".vimrc", "get_cached_app.py"]:
+        for filename in ["common_site_config.json", "supervisord.conf", ".vimrc", "get_cached_app.py", "run_hooks.py"]:
             shutil.copy(os.path.join(self.build_config_path, filename), self.build_directory)
 
         shutil.copytree(
@@ -986,41 +986,27 @@ def download_file(url: str, path: str) -> None:
             for chunk in response.iter_content(chunk_size=1024 * 1024):
                 f.write(chunk)
 
-
-def repack_public_folder(exported: str, tarball: str, app: str) -> None:
-    """Rename public/ to <app>/ as the build unpacks it, leaving out the node_modules bench links in."""
-
-    def rename(name: str) -> str:
-        _, _, rest = name.partition("/")
-        return f"{app}/{rest}" if rest else app
-
-    node_modules = f"{app}/node_modules"
-    with tarfile.open(exported) as source, tarfile.open(tarball, "w:gz") as target:
-        for member in source:
-            name = rename(member.name)
-            if name == node_modules or name.startswith(f"{node_modules}/"):
-                continue
-
-            member.name = name
-            if member.islnk():
-                member.linkname = rename(member.linkname)
-            target.addfile(member, source.extractfile(member) if member.isfile() else None)
-
-
 def upload_app_assets(container: Container, app: str, upload_url: str) -> None:
     with tempfile.TemporaryDirectory() as directory:
-        exported = os.path.join(directory, "public.tar")
-        stream, _ = container.get_archive(f"{IMAGE_BENCH_PATH}/apps/{app}/{app}/public")
-        with open(exported, "wb") as f:
+        # get_archive wraps the file in an (uncompressed) tar, so unwrap it first.
+        outer = os.path.join(directory, "outer.tar")
+        stream, _ = container.get_archive(f"/home/frappe/added_files_{app}.tar.gz")
+        with open(outer, "wb") as f:
             for chunk in stream:
                 f.write(chunk)
 
         tarball = os.path.join(directory, "assets.tar.gz")
-        repack_public_folder(exported, tarball, app)
+        with tarfile.open(outer) as tar:
+            with tar.extractfile(tar.getmembers()[0]) as src, open(tarball, "wb") as dst:
+                shutil.copyfileobj(src, dst)
+
+        with tarfile.open(tarball, "r:gz") as tar:
+            if tar.next() is None:
+                return  # nothing changed
+
         with open(tarball, "rb") as f:
             requests.put(upload_url, data=f, timeout=600).raise_for_status()
 
-
 def get_clone_directory():
     path = os.path.join(os.getcwd(), ".clones")
     if not os.path.exists(path):

agent/tests/test_builder.py

@@ -93,6 +93,7 @@ def test_build_context_includes_the_asset_store_script(self):
             context = SimpleNamespace(build_config_path=build_config_path, build_directory=build_directory)
             ContextManager._copy_build_config_files(context)
             self.assertTrue(os.path.isfile(os.path.join(build_directory, "get_cached_app.py")))
+            self.assertTrue(os.path.isfile(os.path.join(build_directory, "run_hooks.py")))
 
 
 class TestStoredAssetsInBuildContext(unittest.TestCase):