Link ICU data into the Cobalt binary (youtube#3211)

With this change, Cobalt will no longer be deployed with an external ICU
data file that must be loaded at runtime.

b/209049814

Change-Id: I8f67b3176181fa314f3e011ce4cf07fd4a671436

Test-On-Device: true

base/BUILD.gn

@@ -3661,7 +3661,6 @@ test("base_unittests") {
     data_deps += [
       ":base_unittests_bundle_data",
       "//cobalt/network:copy_ssl_certificates",
-      "//third_party/icu:icudata",
     ]
   }
 

cobalt/base/BUILD.gn

@@ -127,8 +127,5 @@ target(gtest_target_type, "base_test") {
     "//testing/gmock",
     "//testing/gtest",
   ]
-  data_deps = [
-    "//cobalt/network:copy_ssl_certificates",
-    "//third_party/icu:icudata",
-  ]
+  data_deps = [ "//cobalt/network:copy_ssl_certificates" ]
 }

cobalt/browser/BUILD.gn

@@ -58,10 +58,7 @@ target(final_executable_type, "cobalt") {
     "//cobalt/css_parser",
     "//net",
   ]
-  data_deps = [
-    "//cobalt/network:copy_ssl_certificates",
-    "//third_party/icu:icudata",
-  ]
+  data_deps = [ "//cobalt/network:copy_ssl_certificates" ]
   if (cobalt_font_package == "empty") {
     data_deps += [ "//cobalt/content/fonts:copy_font_data" ]
   } else {

cobalt/encoding/BUILD.gn

@@ -53,8 +53,5 @@ target(gtest_target_type, "text_encoding_test") {
     "//testing/gtest",
   ]
 
-  data_deps = [
-    "//cobalt/network:copy_ssl_certificates",
-    "//third_party/icu:icudata",
-  ]
+  data_deps = [ "//cobalt/network:copy_ssl_certificates" ]
 }

cobalt/layout_tests/BUILD.gn

@@ -67,7 +67,6 @@ target(gtest_target_type, "layout_tests") {
   data_deps = [
     "//cobalt/layout_tests/testdata:layout_copy_test_data",
     "//cobalt/network:copy_ssl_certificates",
-    "//third_party/icu:icudata",
   ]
   if (cobalt_font_package == "empty") {
     data_deps += [ "//cobalt/content/fonts:copy_font_data" ]

net/BUILD.gn

@@ -3076,7 +3076,6 @@ test("net_unittests") {
       ":net_unittest_files",
       ":third_party_unittest_files",
       "//cobalt/network:copy_ssl_certificates",
-      "//third_party/icu:icudata",
     ]
   }
 

starboard/build/config/BUILDCONFIG.gn

@@ -354,7 +354,6 @@ template("evergreen_loader") {
   # Add additional evergreen_loader target for every native target.
   original_target_name = invoker.original_target_name
   shared_data_deps = invoker.data_deps
-  shared_data_deps += [ "//third_party/icu:icudata" ]
   shared_data_deps += [
     ":copy_loader_app_lib${original_target_name}",
     "//starboard/loader_app:copy_loader_app_content",

starboard/build/config/modular/arm/hardfp/BUILD.gn

@@ -18,6 +18,7 @@ config("sabi_flags") {
     "-target",
     "armv7a-none-eabihf",
   ]
+  asmflags = cflags
 }
 
 config("hardfp") {

starboard/client_porting/eztime/BUILD.gn

@@ -42,8 +42,5 @@ target(gtest_target_type, "eztime_test") {
     "//testing/gtest",
   ]
 
-  data_deps = [
-    "//cobalt/network:copy_ssl_certificates",
-    "//third_party/icu:icudata",
-  ]
+  data_deps = [ "//cobalt/network:copy_ssl_certificates" ]
 }

starboard/doc/evergreen/cobalt_evergreen_overview.md

@@ -486,7 +486,6 @@ Image required for all slot configurations:
 │       └── cobalt <--(SLOT_0)
 │           ├── content <--(relative path defined in kSystemImageContentPath)
 │           │   ├── fonts <--(`empty` configuration)
-│           │   ├── (icu) <--(only present when it needs to be updated by Cobalt Update)
 │           │   ├── licenses
 │           │   ├── ssl
 │           ├── lib
@@ -507,15 +506,13 @@ updates in an example 3-slot configuration:
     ├── installation_2 <--(SLOT_2 - contains new Cobalt version)
     │   ├── content
     │   │   ├── fonts <--(`empty` configuration)
-    │   │   ├── (icu) <--(only present when it needs to be updated by Cobalt Update)
     │   │   ├── licenses
     │   │   ├── ssl
     │   ├── lib
     │   │   └── libcobalt.so <--(SLOT_2 version of libcobalt.so)
     │   ├── manifest.fingerprint
     │   └── manifest.json <-- (Evergreen version information of libcobalt.so under SLOT_2)
     ├── installation_store_<APP_KEY>.pb
-    └── icu (default location shared by installation slots, to be explained below)
 ```
 Note that after the Cobalt binary is loaded by the loader_app, `kSbSystemPathContentDirectory` points to the
 content directory of the running binary, as stated in Starboard Module Reference of system.h.
@@ -559,24 +556,6 @@ On Raspberry Pi the Cobalt fonts are configured the following way:
 <kSbSystemPathContentDirectory>/fonts
 ```
 
-### ICU Tables
-The ICU table should be deployed under the `kSbSystemPathStorageDirectory`. This
-way all Cobalt Evergreen installations would be able to share the same tables.
-The current storage size for the ICU tables is 7MB.
-
-On Raspberry Pi this is:
-
-```
-/home/pi/.cobalt_storage/icu
-```
-The Cobalt Evergreen package will not carry ICU tables by default but may add
-them in the future if needed. When the package has ICU tables they would be
-stored under the content location for the installation:
-
-```
-<SLOT_#>/content/icu
-```
-
 ### Handling Pending Updates
 Pending updates will be picked up on the next application start, which means
 that on platforms that support suspending the platform should check

starboard/elf_loader/BUILD.gn

@@ -93,11 +93,10 @@ if (sb_is_evergreen_compatible && current_toolchain == starboard_toolchain) {
 # TODO: b/309493306 - Stop building evergreen targets for all non-evergreen platforms.
 if (current_toolchain == starboard_toolchain && !is_host_win) {
   target(starboard_level_final_executable_type, "elf_loader_sandbox") {
-    data_deps = [ "//third_party/icu:icudata" ]
     if (cobalt_font_package == "empty") {
-      data_deps += [ "//cobalt/content/fonts:copy_font_data" ]
+      data_deps = [ "//cobalt/content/fonts:copy_font_data" ]
     } else {
-      data_deps += [
+      data_deps = [
         "//cobalt/content/fonts:copy_fonts",
         "//cobalt/content/fonts:fonts_xml",
       ]

starboard/evergreen/arm/softfp/platform_configuration/BUILD.gn

@@ -18,6 +18,7 @@ config("sabi_flags") {
     "-target",
     "armv7a-none-eabi",
   ]
+  asmflags = cflags
 }
 
 config("platform_configuration") {

starboard/examples/window/BUILD.gn

@@ -17,5 +17,4 @@ target(final_executable_type, "starboard_window_example") {
 
   sources = [ "main.cc" ]
   public_deps = [ "//starboard:starboard_group" ]
-  data_deps = [ "//third_party/icu:icudata" ]
 }

starboard/loader_app/BUILD.gn

@@ -90,11 +90,10 @@ if (sb_is_evergreen_compatible && current_toolchain == starboard_toolchain) {
   target(starboard_level_final_executable_type, "loader_app") {
     build_loader = false
     if (target_cpu == "x64" || target_cpu == "arm" || target_cpu == "arm64") {
-      data_deps = [ "//third_party/icu:icudata" ]
       if (cobalt_font_package == "empty") {
-        data_deps += [ "//cobalt/content/fonts:copy_font_data" ]
+        data_deps = [ "//cobalt/content/fonts:copy_font_data" ]
       } else {
-        data_deps += [
+        data_deps = [
           "//cobalt/content/fonts:copy_fonts",
           "//cobalt/content/fonts:fonts_xml",
         ]

starboard/nplb/BUILD.gn

@@ -322,7 +322,6 @@ target(gtest_target_type, "nplb") {
     "//cobalt/network:copy_ssl_certificates",
     "//starboard/nplb/testdata/file_tests:nplb_file_tests_data",
     "//starboard/shared/starboard/player:player_download_test_data",
-    "//third_party/icu:icudata",
   ]
 
   if (is_clang_16) {

starboard/nplb/nplb_evergreen_compat_tests/BUILD.gn

@@ -33,8 +33,5 @@ target(gtest_target_type, "nplb_evergreen_compat_tests") {
     "//testing/gtest",
   ]
 
-  data_deps = [
-    "//cobalt/network:copy_ssl_certificates",
-    "//third_party/icu:icudata",
-  ]
+  data_deps = [ "//cobalt/network:copy_ssl_certificates" ]
 }

starboard/shared/starboard/player/filter/tools/BUILD.gn

@@ -26,7 +26,6 @@ if (current_toolchain == starboard_toolchain) {
     data_deps = [
       "//cobalt/network:copy_ssl_certificates",
       "//starboard/shared/starboard/player:player_download_test_data",
-      "//third_party/icu:icudata",
     ]
   }
 }

third_party/crashpad/crashpad/handler/BUILD.gn

@@ -159,7 +159,6 @@ if (!crashpad_is_ios) {
   crashpad_executable("crashpad_handler") {
     if (crashpad_is_in_starboard) {
       install_target = !crashpad_is_android
-      data_deps = [ "//third_party/icu:icudata" ]
     }
 
     sources = [ "main.cc" ]

third_party/icu/BUILD.gn

@@ -3,6 +3,10 @@
 # found in the LICENSE file.
 
 import("//build/config/host_byteorder.gni")
+if (use_cobalt_customizations) {
+  import("//starboard/build/nasm_assemble.gni")
+  import("//starboard/sabi/sabi.gni")
+}
 import("//third_party/icu/config.gni")
 import("//third_party/icu/sources.gni")
 
@@ -375,6 +379,9 @@ if (is_android && enable_java_templates) {
   }
 }
 
+if (use_cobalt_customizations) {
+  data_dir = "cobalt"
+} else {
 if (is_android) {
   # Use android_small for now to keep the size till we decide to switch to the new one.
   data_dir = "android_small"
@@ -385,13 +392,22 @@ if (is_android) {
 } else {
   data_dir = "common"
 }
+}
 
+if (use_cobalt_customizations) {
+  if (sabi_variables.endianness == "little") {
+    data_bundle_prefix = "icudt68l"
+  } else {
+    data_bundle_prefix = "icudt68b"
+  }
+} else {
 if (current_cpu == "mips" || current_cpu == "mips64" ||
     host_byteorder == "big") {
   data_bundle_prefix = "icudtb"
 } else {
   data_bundle_prefix = "icudtl"
 }
+}
 data_bundle = "${data_bundle_prefix}.dat"
 
 if (icu_use_data_file) {
@@ -423,7 +439,12 @@ if (icu_use_data_file) {
     outputs = [ "$root_out_dir/icudtl_extra.dat" ]
   }
 } else {
+  if (use_cobalt_customizations && (is_win || current_os == "playstation")) {
+    # These platforms use NASM for the data assembly and it expects .asm files.
+    data_assembly = "$target_gen_dir/${data_bundle_prefix}_dat.asm"
+  } else {
   data_assembly = "$target_gen_dir/${data_bundle_prefix}_dat.S"
+  }
   inline_data_assembly = "$target_gen_dir/${data_bundle_prefix}_dat.cc"
   action("make_data_assembly") {
     script = "scripts/make_data_assembly.py"
@@ -436,11 +457,18 @@ if (icu_use_data_file) {
     if (is_mac || is_ios) {
       args += [ "--mac" ]
     } else if (is_win) {
+      if (use_cobalt_customizations) {
+        args += [ "--nasm" ]
+      } else {
       args += [ "--win" ]
+      }
+    } else if (use_cobalt_customizations && current_os == "playstation") {
+      args += [ "--nasm" ]
     }
   }
 
-  if (is_win) {
+  # Cobalt uses MSVC, which doesn't support inline assembly for win32.
+  if (!use_cobalt_customizations && is_win) {
     action("make_inline_data_assembly") {
       deps = [ ":make_data_assembly" ]
       script = "scripts/asm_to_inline_asm.py"
@@ -456,6 +484,12 @@ if (icu_use_data_file) {
     not_needed([ "inline_data_assembly" ])
   }
 
+  if (use_cobalt_customizations && (is_win || current_os == "playstation")) {
+    nasm_assemble("icudata") {
+      sources = [ data_assembly ]
+      deps = [ ":make_data_assembly" ]
+    }
+  } else {
   source_set("icudata") {
     defines = [ "U_HIDE_DATA_SYMBOL" ]
     if (is_win) {
@@ -466,4 +500,5 @@ if (icu_use_data_file) {
       deps = [ ":make_data_assembly" ]
     }
   }
+  }
 }

third_party/icu/README.cobalt

@@ -29,71 +29,27 @@ This is based off of section B in the README.chromium file.
 5. Run `$ICU_ROOT/scripts/clean_up_data_source.sh` to clean up the changes
    made to $ICU_ROOT from the call to `patch_locale.sh`.
 
-Cobalt uses a packaged ICU data file instead of individual files, but
-instructions for both options have been included.
-
-***Option 1: Rebuild packaged ICU data file
+Cobalt uses a packaged ICU data file instead of individual files. This packaged
+ICU data file is used during the build to link ICU data into the Cobalt binary.
 
 6. Replace the files in this directory with the new files that you have built.
 
 Run:
 
     ```
-    cd $COBALT_SRC_ROOT/src/cobalt/content/icu
+    cd $COBALT_SRC_ROOT/third_party/icu/cobalt
     rm -rf *
     cp $ICU_DATA_ROOT/data/out/tmp/icudt68l.dat .
     cp $ICU_DATA_ROOT/data/out/tmp/icudt68b.dat .
     ```
 
 Note that at the time of this writing, the final resulting contents had a size
 of 4.6MB, from running the command
-`du -sh $COBALT_SRC_ROOT/src/cobalt/content/icu/icudt68l.dat`.
-
-***Option 2: Rebuild individual ICU data files
-
-6. Create a new directory where we will rebuild ICU with different options
-    because the options used above will produce a version of the tool `icupkg` (
-    which we need to use to translate from little to big endian) that does not
-    work.  Call this new directory $ICU_PKG_ROOT.
-
-7. Change directory to $ICU_PKG_ROOT.
-
-8. Run
-   ```
-   $ICU_ROOT/source/runConfigureICU --enable-debug --disable-release \
-     Linux/gcc  --disable-tests --disable-layoutex
-   ```
-
-9. Run `make -j 120`
-
-10. Run `export LD_LIBRARY_PATH=$ICU_PKG_ROOT/lib`
-
-11. Run `sudo apt install icu-devtools` to obtain the `icupkg` command and its
-    dependencies.  Even though you will be running a locally-built version of
-    `icupkg`, you will need the shared libraries installed by this command.
-
-12. Replace the files in this directory with the new files that you have built.
-    They will exist as a little-endian package file, but you can use the
-    `icupkg` tool to both unpack the files out of the package (which Cobalt
-    prefers), and switch from little-endian to big-endian.
-
-    Run:
-
-    ```
-    cd $COBALT_SRC_ROOT/src/cobalt/content/icu
-    rm -rf *
-    mkdir icudt68l icudt68b
-    $ICU_PKG_ROOT/bin/icupkg --ignore-deps -tl -x '*' -d icudt68l $ICU_DATA_ROOT/data/out/tmp/icudt68l.dat
-    $ICU_PKG_ROOT/bin/icupkg --ignore-deps -tb -x '*' -d icudt68b $ICU_DATA_ROOT/data/out/tmp/icudt68l.dat
-    ```
-
-Note that at the time of this writing, the final resulting contents had a size
-of 6.7MB, from running the command
-`du -sh $COBALT_SRC_ROOT/src/cobalt/content/icu/icudt68l`.
+`du -sh $COBALT_SRC_ROOT/third_party/icu/cobalt/icudt68l.dat`.
 
 ********************************************************************************
 
 You should now have the ICU data files generated and ready to go in the Cobalt
 source tree, and you can clean up any outstanding files.  If there are any
 leftover changes in your $ICU_ROOT directory, you can revert them.  At this
-point you can re-gyp, re-ninja, and run Cobalt and it should work.
+point you can run `gn gen`, run `ninja`, and run Cobalt and it should work.

third_party/icu/config.gni

@@ -5,7 +5,11 @@
 declare_args() {
   # Tells icu to load an external data file rather than rely on the icudata
   # being linked directly into the binary.
-  icu_use_data_file = true
+  if (use_cobalt_customizations) {
+    icu_use_data_file = false
+  } else {
+    icu_use_data_file = true
+  }
 
   # If true, compile icu into a standalone static library. Currently this is
   # only useful on Chrome OS.