[Frontend] Switch from YAML to .strings based localization

xedin · xedin · commit 5a164c5f8aaa · 2022-08-22T10:23:06.000-07:00
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
@@ -1450,7 +1450,7 @@ static bool ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
       // for the specified locale code.
       llvm::SmallString<128> localizationPath(A->getValue());
       llvm::sys::path::append(localizationPath, Opts.LocalizationCode);
-      llvm::sys::path::replace_extension(localizationPath, ".yaml");
+      llvm::sys::path::replace_extension(localizationPath, ".strings");
       if (!llvm::sys::fs::exists(localizationPath)) {
         Diags.diagnose(SourceLoc(), diag::warning_cannot_find_locale_file,
                        Opts.LocalizationCode, localizationPath);
diff --git a/localization/CMakeLists.txt b/localization/CMakeLists.txt
@@ -6,16 +6,16 @@ add_custom_command(
   COMMAND
     ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/diagnostics/ ${CMAKE_BINARY_DIR}/share/swift/diagnostics/
   COMMAND
-    "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-def-to-yaml-converter"
+    "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-def-to-strings-converter"
       --output-directory ${CMAKE_BINARY_DIR}/share/swift/diagnostics/
   COMMAND
     "${SWIFT_NATIVE_SWIFT_TOOLS_PATH}/swift-serialize-diagnostics"
-      --input-file-path ${CMAKE_BINARY_DIR}/share/swift/diagnostics/en.yaml
+      --input-file-path ${CMAKE_BINARY_DIR}/share/swift/diagnostics/en.strings
       --output-directory ${CMAKE_BINARY_DIR}/share/swift/diagnostics/
   COMMAND
     ${CMAKE_COMMAND} -E touch ${diagnostic_witness}
   DEPENDS
-    swift-def-to-yaml-converter
+    swift-def-to-strings-converter
     swift-serialize-diagnostics
     # Add files in diagnostics subdirectory when they're created
 )
@@ -31,4 +31,5 @@ swift_install_in_component(
   FILES_MATCHING
   PATTERN "*.db"
   PATTERN "*.yaml"
+  PATTERN "*.strings"
 )
diff --git a/test/diagnostics/Localization/Inputs/en.strings b/test/diagnostics/Localization/Inputs/en.strings
@@ -0,0 +1,25 @@
+/**
+ *--- en.strings - Localized diagnostic messages for English --------------===*
+ *
+ * This source file is part of the Swift.org open source project
+ *
+ * Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+ * Licensed under Apache License v2.0 with Runtime Library Exception
+ *
+ * See https://swift.org/LICENSE.txt for license information
+ * See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ * This file defines the diagnostic messages for the English language.
+ * Each diagnostic is described in the following format:
+ *   "<diagnostic-id>" = "<diagnostic-message>";
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ */
+"lex_unterminated_string" = "unterminated string literal";
+"var_init_self_referential" = "variable used within its own initial value";
+/* Tests different number of spaces between id and diagnostic message */
+"cannot_find_in_scope"=     "cannot %select{find|find operator}1 %0 in scope";
+"warning_invalid_locale_code" = "unsupported locale code; supported locale codes are '%0'";
diff --git a/test/diagnostics/Localization/Inputs/en.yaml b/test/diagnostics/Localization/Inputs/en.yaml
diff --git a/test/diagnostics/Localization/Inputs/fr.strings b/test/diagnostics/Localization/Inputs/fr.strings
@@ -0,0 +1,27 @@
+/**
+ *===--- fr.strings - Localized diagnostic messages for French ------------===*
+ *
+ * This source file is part of the Swift.org open source project
+ *
+ * Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+ * Licensed under Apache License v2.0 with Runtime Library Exception
+ *
+ * See https://swift.org/LICENSE.txt for license information
+ * See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ * This file defines the diagnostic messages for the French language.
+ * Each diagnostic is described in the following format:
+ *  "<diagnostic-id>" = "<diagnostic-message>";
+ *
+ */
+
+/* Different order from `en.strings` */
+"var_init_self_referential" = "variable utilisée dans sa propre valeur initiale";
+
+"lex_unterminated_string" = "chaîne non terminée littérale";
+
+"cannot_find_in_scope" = "impossible %select{de trouver|de trouver opérateur}1 %0 portée";
+
+"warning_invalid_locale_code" = "code de paramètres régionaux non pris en charge; les codes pris en charge sont '%0'";
diff --git a/test/diagnostics/Localization/Inputs/fr.yaml b/test/diagnostics/Localization/Inputs/fr.yaml
diff --git a/test/diagnostics/Localization/fr_debug_diagnostic_name.swift b/test/diagnostics/Localization/fr_debug_diagnostic_name.swift
@@ -1,6 +1,6 @@
 // RUN: %empty-directory(%t)
-// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.yaml --output-directory=%t/
-// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.yaml --output-directory=%t/
+// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.strings --output-directory=%t/
+// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.strings --output-directory=%t/
 // RUN: not %target-swift-frontend -debug-diagnostic-names -localization-path %S/Inputs -locale fr -typecheck %s 2>&1 | %FileCheck %s --check-prefix=CHECK_NAMES
 
 _ = "HI!
diff --git a/test/diagnostics/Localization/fr_localization.swift b/test/diagnostics/Localization/fr_localization.swift
@@ -1,6 +1,6 @@
 // RUN: %empty-directory(%t)
-// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.yaml --output-directory=%t/
-// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.yaml --output-directory=%t/
+// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/fr.strings --output-directory=%t/
+// RUN: swift-serialize-diagnostics --input-file-path=%S/Inputs/en.strings --output-directory=%t/
 // RUN: %target-typecheck-verify-swift -localization-path %t -locale fr
 
 _ = "HI!
diff --git a/test/diagnostics/Localization/no_localization_files_and_wrong_path.swift b/test/diagnostics/Localization/no_localization_files_and_wrong_path.swift
@@ -1,6 +1,6 @@
 // RUN: %target-typecheck-verify-swift -localization-path /Nonexistent_path -locale en
 
-// <unknown>:0: warning: cannot find translations for 'en' at '/Nonexistent_path/en.yaml': no such file
+// <unknown>:0: warning: cannot find translations for 'en' at '/Nonexistent_path/en.strings': no such file
 // <unknown>:0: warning: specified localization directory '/Nonexistent_path' does not exist, translation is disabled
 
 _ = "HI!
diff --git a/tools/swift-serialize-diagnostics/swift-serialize-diagnostics.cpp b/tools/swift-serialize-diagnostics/swift-serialize-diagnostics.cpp
@@ -42,7 +42,7 @@ static llvm::cl::OptionCategory Category("swift-serialize-diagnostics Options");
 
 static llvm::cl::opt<std::string>
     InputFilePath("input-file-path",
-                  llvm::cl::desc("Path to the YAML input file"),
+                  llvm::cl::desc("Path to the YAML or `.strings` input file"),
                   llvm::cl::cat(Category));
 
 static llvm::cl::opt<std::string>
@@ -60,38 +60,50 @@ int main(int argc, char *argv[]) {
                                     "Swift Serialize Diagnostics Tool\n");
 
   if (!llvm::sys::fs::exists(options::InputFilePath)) {
-    llvm::errs() << "YAML file not found\n";
+    llvm::errs() << "diagnostics file not found\n";
     return EXIT_FAILURE;
   }
 
-  YAMLLocalizationProducer yaml(options::InputFilePath);
-
   auto localeCode = llvm::sys::path::filename(options::InputFilePath);
   llvm::SmallString<128> SerializedFilePath(options::OutputDirectory);
   llvm::sys::path::append(SerializedFilePath, localeCode);
   llvm::sys::path::replace_extension(SerializedFilePath, ".db");
 
   SerializedLocalizationWriter Serializer;
-  yaml.forEachAvailable(
-      [&Serializer](swift::DiagID id, llvm::StringRef translation) {
-        Serializer.insert(id, translation);
-      });
+
+  if (llvm::sys::path::extension(options::InputFilePath) == ".yaml") {
+    YAMLLocalizationProducer yaml(options::InputFilePath);
+
+    yaml.forEachAvailable(
+        [&Serializer](swift::DiagID id, llvm::StringRef translation) {
+          Serializer.insert(id, translation);
+        });
+
+    // Print out the diagnostics IDs that are available in YAML but not
+    // available in `.def`
+    if (!yaml.unknownIDs.empty()) {
+      llvm::errs() << "These diagnostic IDs are no longer available: '";
+      llvm::interleave(
+          yaml.unknownIDs, [&](std::string id) { llvm::errs() << id; },
+          [&] { llvm::errs() << ", "; });
+      llvm::errs() << "'\n";
+    }
+  } else {
+    assert(llvm::sys::path::extension(options::InputFilePath) == ".strings");
+
+    StringsLocalizationProducer strings(options::InputFilePath);
+
+    strings.forEachAvailable(
+        [&Serializer](swift::DiagID id, llvm::StringRef translation) {
+          Serializer.insert(id, translation);
+        });
+  }
 
   if (Serializer.emit(SerializedFilePath.str())) {
     llvm::errs() << "Cannot serialize diagnostic file "
                  << options::InputFilePath << '\n';
     return EXIT_FAILURE;
   }
 
-  // Print out the diagnostics IDs that are available in YAML but not available
-  // in `.def`
-  if (!yaml.unknownIDs.empty()) {
-    llvm::errs() << "These diagnostic IDs are no longer available: '";
-    llvm::interleave(
-        yaml.unknownIDs, [&](std::string id) { llvm::errs() << id; },
-        [&] { llvm::errs() << ", "; });
-    llvm::errs() << "'\n";
-  }
-
   return EXIT_SUCCESS;
 }
diff --git a/unittests/Localization/CMakeLists.txt b/unittests/Localization/CMakeLists.txt
@@ -1,5 +1,5 @@
 add_swift_unittest(swiftLocalizationTests
-  DefToYAMLConverterTests.cpp
+  DefToStringsConverterTests.cpp
   SerializationTests.cpp)
 
 target_link_libraries(swiftLocalizationTests
diff --git a/unittests/Localization/DefToStringsConverterTests.cpp b/unittests/Localization/DefToStringsConverterTests.cpp
@@ -1,8 +1,8 @@
-//===--- DefToYAMLConverterTests.cpp -------------------------------------===//
+//===--- DefToStringsConverterTests.cpp -----------------------------------===//
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
@@ -49,22 +49,22 @@ TEST_F(LocalizationTest, MissingLocalizationFiles) {
   ASSERT_TRUE(llvm::sys::fs::exists(getDefaultLocalizationPath()));
   llvm::SmallString<128> EnglishLocalization(getDefaultLocalizationPath());
   llvm::sys::path::append(EnglishLocalization, "en");
-  llvm::sys::path::replace_extension(EnglishLocalization, ".yaml");
+  llvm::sys::path::replace_extension(EnglishLocalization, ".strings");
   ASSERT_TRUE(llvm::sys::fs::exists(EnglishLocalization));
   llvm::sys::path::replace_extension(EnglishLocalization, ".db");
   ASSERT_TRUE(llvm::sys::fs::exists(EnglishLocalization));
 }
 
 TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesSequentially) {
-  YAMLLocalizationProducer yaml(YAMLPath);
-  yaml.forEachAvailable([](swift::DiagID id, llvm::StringRef translation) {
+  StringsLocalizationProducer strings(DiagsPath);
+  strings.forEachAvailable([](swift::DiagID id, llvm::StringRef translation) {
     llvm::StringRef msg = diagnosticMessages[static_cast<uint32_t>(id)];
     ASSERT_EQ(msg, translation);
   });
 }
 
 TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesRandomly) {
-  YAMLLocalizationProducer yaml(YAMLPath);
+  StringsLocalizationProducer strings(DiagsPath);
 
   std::random_device rd;
   std::mt19937 gen(rd());
@@ -74,7 +74,7 @@ TEST_F(LocalizationTest, ConverterTestMatchDiagnosticMessagesRandomly) {
     unsigned randomNum = RandNumber(LocalDiagID::NumDiags);
     DiagID randomId = static_cast<DiagID>(randomNum);
     llvm::StringRef msg = diagnosticMessages[randomNum];
-    llvm::StringRef translation = yaml.getMessageOr(randomId, "");
+    llvm::StringRef translation = strings.getMessageOr(randomId, "");
     ASSERT_EQ(msg, translation);
   }
 }
diff --git a/unittests/Localization/LocalizationTest.h b/unittests/Localization/LocalizationTest.h
@@ -53,15 +53,15 @@ struct LocalizationTest : public ::testing::Test {
   llvm::SmallVector<std::string, 4> TempFiles;
 
 public:
-  std::string YAMLPath;
+  std::string DiagsPath;
 
   LocalizationTest() {
-    YAMLPath = std::string(createTemporaryFile("en", "yaml"));
+    DiagsPath = std::string(createTemporaryFile("en", "strings"));
   }
 
   void SetUp() override {
-    bool failed = convertDefIntoYAML(YAMLPath);
-    assert(!failed && "failed to generate a YAML file");
+    bool failed = convertDefIntoStrings(DiagsPath);
+    assert(!failed && "failed to generate a `.strings` file");
   }
 
   void TearDown() override {
@@ -85,7 +85,7 @@ struct LocalizationTest : public ::testing::Test {
   unsigned RandNumber(unsigned n) { return unsigned(rand()) % n; }
 
 protected:
-  static bool convertDefIntoYAML(std::string outputPath) {
+  static bool convertDefIntoStrings(std::string outputPath) {
     std::error_code error;
     llvm::raw_fd_ostream OS(outputPath, error, llvm::sys::fs::OF_None);
     if (OS.has_error() || error)
@@ -95,7 +95,7 @@ struct LocalizationTest : public ::testing::Test {
     llvm::ArrayRef<const char *> messages(diagnosticMessages,
                                           LocalDiagID::NumDiags);
 
-    DefToYAMLConverter converter(ids, messages);
+    DefToStringsConverter converter(ids, messages);
     converter.convert(OS);
 
     OS.flush();
diff --git a/unittests/Localization/SerializationTests.cpp b/unittests/Localization/SerializationTests.cpp
