compute empty files in c++ binary

Author: tobyh-canva

python/private/py_executable.bzl

@@ -832,7 +832,9 @@ def _create_zip_file(ctx, *, output, zip_main, runfiles):
         args.add("--repo-mapping-manifest", zip_repo_mapping_manifest.path)
         inputs.append(zip_repo_mapping_manifest)
 
-    args.add_all(runfiles.empty_filenames, map_each=_get_zip_empty_path_arg)
+    # Empty __init__.py files are now generated by the C++ binary
+    # based on the directories containing Python files. This eliminates
+    # the expensive flattening of the files depset during analysis.
     args.add_all(runfiles.files, map_each=_get_zip_path_arg)
 
     ctx.actions.run(
@@ -846,10 +848,6 @@ def _create_zip_file(ctx, *, output, zip_main, runfiles):
     )
 
 
-def _get_zip_empty_path_arg(file):
-    return "{}=".format(file.short_path)
-
-
 def _get_zip_path_arg(file):
     return "{}={}".format(file.short_path, file.path)
 

python/private/py_executable_zip_gen.cc

@@ -29,6 +29,7 @@
 #include <fstream>
 #include <iostream>
 #include <memory>
+#include <set>
 #include <string>
 #include <vector>
 
@@ -68,24 +69,6 @@ bool starts_with(const std::string& str, const std::string& prefix) {
 
 } // namespace path
 
-// Get zip runfiles path for a file
-std::string get_zip_runfiles_path(const std::string& path,
-                                   const std::string& workspace_name,
-                                   bool legacy_external_runfiles) {
-  std::string zip_runfiles_path;
-  
-  if (legacy_external_runfiles && path::starts_with(path, "external/")) {
-    // Remove "external/" prefix
-    zip_runfiles_path = path::relativize(path, "external");
-  } else {
-    // Normalize workspace_name/path
-    std::string combined = workspace_name + "/" + path;
-    zip_runfiles_path = path::normalize(combined);
-  }
-  
-  return "runfiles/" + zip_runfiles_path;
-}
-
 // Parse a file entry in "short_path=disk_path" or "short_path=" format
 struct FileEntry {
   std::string short_path;
@@ -108,6 +91,84 @@ struct FileEntry {
   }
 };
 
+// Get zip runfiles path for a file
+std::string get_zip_runfiles_path(const std::string& path,
+                                   const std::string& workspace_name,
+                                   bool legacy_external_runfiles) {
+  std::string zip_runfiles_path;
+  
+  if (legacy_external_runfiles && path::starts_with(path, "external/")) {
+    // Remove "external/" prefix
+    zip_runfiles_path = path::relativize(path, "external");
+  } else {
+    // Normalize workspace_name/path
+    std::string combined = workspace_name + "/" + path;
+    zip_runfiles_path = path::normalize(combined);
+  }
+  
+  return "runfiles/" + zip_runfiles_path;
+}
+
+// Check if a path is a Python file (.py, .pyc, .so, .pyd)
+bool is_python_file(const std::string& path) {
+  if (path.empty()) return false;
+  
+  // Check for Python file extensions
+  if (path.size() >= 3 && path.substr(path.size() - 3) == ".py") return true;
+  if (path.size() >= 4 && path.substr(path.size() - 4) == ".pyc") return true;
+  if (path.size() >= 3 && path.substr(path.size() - 3) == ".so") return true;
+  if (path.size() >= 4 && path.substr(path.size() - 4) == ".pyd") return true;
+  
+  return false;
+}
+
+// Extract directory path from a file path
+std::string get_directory(const std::string& path) {
+  size_t last_slash = path.find_last_of('/');
+  if (last_slash == std::string::npos) {
+    return "";
+  }
+  return path.substr(0, last_slash);
+}
+
+// Generate all parent directories (excluding root)
+std::vector<std::string> get_parent_directories(const std::string& dir) {
+  std::vector<std::string> parents;
+  std::string current = dir;
+  
+  while (!current.empty()) {
+    parents.push_back(current);
+    size_t last_slash = current.find_last_of('/');
+    if (last_slash == std::string::npos) {
+      break;
+    }
+    current = current.substr(0, last_slash);
+  }
+  
+  return parents;
+}
+
+// Collect all directories that need __init__.py files
+// This replicates the logic of Bazel's EmptyFilesSupplier Java class
+std::set<std::string> collect_init_directories(const std::vector<FileEntry>& files) {
+  std::set<std::string> init_dirs;
+  
+  // For each Python file, add its directory and all parent directories
+  for (const auto& file : files) {
+    if (is_python_file(file.short_path)) {
+      std::string dir = get_directory(file.short_path);
+      if (!dir.empty()) {
+        // Add all parent directories (this replicates EmptyFilesSupplier logic)
+        for (const auto& parent : get_parent_directories(dir)) {
+          init_dirs.insert(parent);
+        }
+      }
+    }
+  }
+  
+  return init_dirs;
+}
+
 int main(int argc, char* argv[]) {
   if (argc != 2) {
     std::cerr << "Usage: " << argv[0] << " <params_file>" << std::endl;
@@ -194,6 +255,10 @@ int main(int argc, char* argv[]) {
     return 1;
   }
 
+  // Collect directories that need __init__.py files
+  // This replicates the logic of Bazel's EmptyFilesSupplier Java class
+  std::set<std::string> init_dirs = collect_init_directories(files);
+  
   // Generate zip manifest
   // Order must match main branch for reproducible builds
   std::string manifest_file = "zip_manifest.txt";
@@ -206,17 +271,25 @@ int main(int argc, char* argv[]) {
   // 1. Main file
   manifest << "__main__.py=" << main_file << "\n";
 
-  // 2. Default empty files
+  // 2. Default empty files at root
   manifest << "__init__.py=\n";
   manifest << get_zip_runfiles_path("__init__.py", workspace_name, legacy_external_runfiles) << "=\n";
 
-  // 3. Process file entries
+  // 3. Generate empty __init__.py files for all directories containing Python files
+  // This replicates the EmptyFilesSupplier logic from Bazel's Java code
+  for (const auto& dir : init_dirs) {
+    std::string init_path = dir + "/__init__.py";
+    std::string zip_path = get_zip_runfiles_path(init_path, workspace_name, legacy_external_runfiles);
+    manifest << zip_path << "=\n";
+  }
+  
+  // 4. Process file entries
   for (const auto& file : files) {
     std::string zip_path = get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles);
     manifest << zip_path << "=" << file.disk_path << "\n";
   }
 
-  // 4. Repo mapping manifest (last, to match main branch order)
+  // 5. Repo mapping manifest (last, to match main branch order)
   if (!repo_mapping_manifest.empty()) {
     manifest << "runfiles/_repo_mapping=" << repo_mapping_manifest << "\n";
   }