Refactor: Move io to gemma/io and fix build paths

.github/workflows/build.yml

@@ -77,4 +77,4 @@ jobs:
         with:
           path: ~/.cache/bazel
           key: bazel-${{ runner.os }}
-      - run: bazel build --cxxopt=-std=c++20 //:gemma --jobs=10 --show_progress_rate_limit=1
+      - run: bazel build --cxxopt=-std=c++20 //:gemma_main --jobs=10 --show_progress_rate_limit=1

BUILD.bazel

@@ -55,7 +55,7 @@ cc_library(
     hdrs = ["util/args.h"],
     deps = [
         ":basics",
-        "//io",  # Path
+        "//gemma/io",  # Path
         "@highway//:hwy",
     ],
 )
@@ -112,7 +112,7 @@ cc_library(
         ":threading",
         ":topology",
         ":zones",
-        "//io",
+        "//gemma/io",
         "@highway//:hwy",
         "@highway//:hwy_test_util",
         "@highway//:profiler",
@@ -192,8 +192,8 @@ cc_library(
     deps = [
         ":basics",
         "//compression:types",
-        "//io",
-        "//io:fields",
+        "//gemma/io",
+        "//gemma/io:fields",
         "@highway//:hwy",  # base.h
     ],
 )
@@ -205,7 +205,7 @@ cc_test(
         ":configs",
         "@googletest//:gtest_main",  # buildcleaner: keep
         "//compression:types",
-        "//io:fields",
+        "//gemma/io:fields",
     ],
 )
 
@@ -230,7 +230,7 @@ cc_library(
         ":tensor_info",
         ":threading_context",
         "//compression:types",
-        "//io:fields",
+        "//gemma/io:fields",
         "@highway//:hwy",
         "@highway//:profiler",
     ],
@@ -261,9 +261,9 @@ cc_library(
         ":threading_context",
         ":tokenizer",
         "//compression:types",
-        "//io",
-        "//io:blob_store",
-        "//io:fields",
+        "//gemma/io",
+        "//gemma/io:blob_store",
+        "//gemma/io:fields",
         "@highway//:hwy",
         "@highway//:profiler",
     ],
@@ -282,7 +282,7 @@ cc_library(
         ":threading_context",
         ":zones",
         "//compression:compress",
-        "//io:blob_store",
+        "//gemma/io:blob_store",
         "@highway//:hwy",
         "@highway//:profiler",
     ],
@@ -577,7 +577,7 @@ cc_library(
         ":mat",
         ":threading_context",
         "//compression:types",
-        "//io",
+        "//gemma/io",
         "@highway//:hwy",
         "@highway//:profiler",
     ],
@@ -641,7 +641,7 @@ cc_library(
         ":threading_context",
         ":zones",
         "//compression:compress",
-        "//io",
+        "//gemma/io",
         "@highway//:hwy",
         "@highway//:profiler",
         "@highway//:stats",
@@ -681,8 +681,8 @@ cc_library(
         ":zones",
         "//compression:compress",
         "//compression:types",
-        "//io",
-        "//io:blob_store",
+        "//gemma/io",
+        "//gemma/io:blob_store",
         "//paligemma:image",
         "@highway//:hwy",
         "@highway//:nanobenchmark",  # timer
@@ -795,7 +795,7 @@ cc_test(
         ":configs",
         ":gemma_lib",
         "@googletest//:gtest_main",  # buildcleaner: keep
-        "//io",
+        "//gemma/io",
         "@highway//:hwy",
         "@highway//:hwy_test_util",
     ],
@@ -823,7 +823,7 @@ cc_test(
 )
 
 cc_binary(
-    name = "gemma",
+    name = "gemma_main",
     srcs = ["gemma/run.cc"],
     deps = [
         ":args",
@@ -847,7 +847,7 @@ cc_binary(
         ":benchmark_helper",
         ":cross_entropy",
         ":gemma_lib",
-        "//io",
+        "//gemma/io",
         "@highway//:hwy",
         "@highway//:nanobenchmark",
         "@nlohmann_json//:json",
@@ -874,7 +874,7 @@ cc_binary(
         ":args",
         ":benchmark_helper",
         ":gemma_lib",
-        "//io",
+        "//gemma/io",
         "@highway//:hwy",
         "@nlohmann_json//:json",
     ],
@@ -887,7 +887,7 @@ cc_binary(
         ":args",
         ":benchmark_helper",
         ":gemma_lib",
-        "//io",
+        "//gemma/io",
         "@highway//:hwy",
         "@highway//:profiler",
         "@nlohmann_json//:json",

CMakeLists.txt

@@ -99,13 +99,13 @@ set(SOURCES
   gemma/vit.h
   gemma/weights.cc
   gemma/weights.h
-  io/blob_store.cc
-  io/blob_store.h
-  io/fields.cc
-  io/fields.h
-  io/io_win.cc
-  io/io.cc
-  io/io.h
+  gemma/io/blob_store.cc
+  gemma/io/blob_store.h
+  gemma/io/fields.cc
+  gemma/io/fields.h
+  gemma/io/io_win.cc
+  gemma/io/io.cc
+  gemma/io/io.h
   ops/dot-inl.h
   ops/matmul_static_bf16.cc
   ops/matmul_static_f32.cc
@@ -225,8 +225,8 @@ set(GEMMA_TEST_FILES
   gemma/gemma_args_test.cc
   gemma/flash_attention_test.cc
   gemma/tensor_info_test.cc
-  io/blob_store_test.cc
-  io/fields_test.cc
+  gemma/io/blob_store_test.cc
+  gemma/io/fields_test.cc
   ops/bench_matmul.cc
   ops/dot_test.cc
   ops/matmul_test.cc
@@ -259,7 +259,7 @@ endif()  # GEMMA_ENABLE_TESTS
 
 ## Tools
 
-add_executable(migrate_weights io/migrate_weights.cc)
+add_executable(migrate_weights gemma/io/migrate_weights.cc)
 target_link_libraries(migrate_weights libgemma hwy hwy_contrib)
 
 

compression/python/BUILD.bazel

@@ -23,8 +23,8 @@ cc_library(
         "//:threading_context",
         "//:tokenizer",
         "//compression:compress",
-        "//io",
-        "//io:blob_store",
+        "//gemma/io",
+        "//gemma/io:blob_store",
         "@highway//:hwy",
     ],
 )

compression/python/compression_clif_aux.cc

@@ -29,19 +29,20 @@
 
 #include "compression/compress.h"  // ScaleWeights
 #include "gemma/configs.h"         // ModelConfig
+#include "gemma/io/blob_store.h"   // BlobWriter
+#include "gemma/io/io.h"           // Path
 #include "gemma/model_store.h"     // ModelStore
 #include "gemma/tensor_info.h"     // TensorInfo
 #include "gemma/tokenizer.h"
-#include "io/blob_store.h"  // BlobWriter
-#include "io/io.h"          // Path
 #include "util/basics.h"
 #include "util/mat.h"
 #include "util/threading_context.h"
 
+
 #undef HWY_TARGET_INCLUDE
 #define HWY_TARGET_INCLUDE \
   "compression/python/compression_clif_aux.cc"  // NOLINT
-#include "hwy/foreach_target.h"  // IWYU pragma: keep
+#include "hwy/foreach_target.h"                 // IWYU pragma: keep
 #include "hwy/highway.h"
 // After highway.h
 #include "compression/compress-inl.h"

compression/python/compression_clif_aux.h

@@ -23,11 +23,12 @@
 
 #include "compression/types.h"  // Type
 #include "gemma/configs.h"
+#include "gemma/io/blob_store.h"
 #include "gemma/model_store.h"
 #include "gemma/tensor_info.h"
-#include "io/blob_store.h"
-#include "util/mat.h"
 #include "hwy/aligned_allocator.h"  // Span
+#include "util/mat.h"
+
 
 namespace gcpp {
 

evals/benchmark.cc

@@ -11,11 +11,12 @@
 #include "evals/benchmark_helper.h"
 #include "evals/cross_entropy.h"
 #include "gemma/gemma.h"
-#include "io/io.h"  // Path
-#include "util/args.h"
+#include "gemma/io/io.h"  // Path
 #include "hwy/base.h"
 #include "hwy/timer.h"
 #include "nlohmann/json.hpp"
+#include "util/args.h"
+
 
 namespace gcpp {
 
@@ -85,8 +86,8 @@ int BenchmarkCrossEntropy(GemmaEnv& env, const Path& text,
     LogSpeedStats(time_start, pos + num_tokens);
     std::string text_slice = env.StringFromTokens(prompt_slice);
     total_input_len += text_slice.size();
-    printf("Total cross entropy: %f [cumulative: %f]\n",
-           entropy, total_entropy);
+    printf("Total cross entropy: %f [cumulative: %f]\n", entropy,
+           total_entropy);
     printf("Cross entropy per byte: %f [cumulative: %f]\n",
            entropy / text_slice.size(), total_entropy / total_input_len);
   }

evals/debug_prompt.cc

@@ -20,10 +20,11 @@
 
 #include "evals/benchmark_helper.h"
 #include "gemma/gemma.h"  // LayersOutputFunc
-#include "io/io.h"
-#include "util/args.h"
+#include "gemma/io/io.h"
 #include "hwy/base.h"
 #include "nlohmann/json.hpp"
+#include "util/args.h"
+
 
 using json = nlohmann::json;
 

evals/run_mmlu.cc

@@ -21,12 +21,13 @@
 
 #include "evals/benchmark_helper.h"
 #include "gemma/gemma.h"  // Gemma
-#include "io/io.h"        // Path
-#include "util/args.h"
+#include "gemma/io/io.h"  // Path
 #include "hwy/base.h"
 #include "hwy/highway.h"
 #include "hwy/profiler.h"
 #include "nlohmann/json.hpp"
+#include "util/args.h"
+
 
 namespace gcpp {
 

gemma/configs.cc

@@ -22,8 +22,8 @@
 #include <vector>
 
 #include "compression/types.h"  // Type
-#include "io/fields.h"          // IFields
-#include "io/io.h"              // Path
+#include "gemma/io/fields.h"          // IFields
+#include "gemma/io/io.h"              // Path
 #include "hwy/base.h"
 
 namespace gcpp {

gemma/configs.h

@@ -26,8 +26,8 @@
 #include <vector>
 
 #include "compression/types.h"  // Type
-#include "io/fields.h"          // IFieldsVisitor
-#include "io/io.h"              // Path
+#include "gemma/io/fields.h"          // IFieldsVisitor
+#include "gemma/io/io.h"              // Path
 #include "util/basics.h"
 
 namespace gcpp {

gemma/configs_test.cc

@@ -7,7 +7,7 @@
 
 #include "gtest/gtest.h"
 #include "compression/types.h"  // Type
-#include "io/fields.h"           // Type
+#include "gemma/io/fields.h"           // Type
 
 namespace gcpp {
 

gemma/gemma.cc

@@ -58,8 +58,8 @@
 #include "gemma/configs.h"
 #include "gemma/model_store.h"
 #include "gemma/weights.h"
-#include "io/blob_store.h"
-#include "io/io.h"  // Path
+#include "gemma/io/blob_store.h"
+#include "gemma/io/io.h"  // Path
 #include "ops/matmul.h"
 #include "paligemma/image.h"
 #include "util/basics.h"

gemma/gemma.h

@@ -29,8 +29,8 @@
 #include "gemma/model_store.h"
 #include "gemma/query.h"
 #include "gemma/weights.h"
-#include "io/blob_store.h"
-#include "io/io.h"       // Path
+#include "gemma/io/blob_store.h"
+#include "gemma/io/io.h"       // Path
 #include "ops/matmul.h"  // MatMulEnv
 #include "paligemma/image.h"
 #include "util/basics.h"  // TokenAndProb

gemma/gemma_args.h

@@ -27,7 +27,7 @@
 
 #include "compression/types.h"
 #include "gemma/configs.h"
-#include "io/io.h"        // Path
+#include "gemma/io/io.h"        // Path
 #include "util/args.h"    // IWYU pragma: export
 #include "util/basics.h"  // Tristate
 #include "util/mat.h"