// Copyright 2025 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <fstream>
#include <map>
#include <optional>
#include <queue>
#include <set>
#include <string>
#include <vector>

#include "include/nlohmann/json.hpp"
#include "pugixml.hpp"
#include "absl/algorithm/container.h"
#include "absl/flags/flag.h"
#include "absl/log/check.h"
#include "absl/log/log.h"
#include "absl/strings/match.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/str_split.h"
#include "absl/strings/string_view.h"

ABSL_FLAG(std::vector<std::string>, target_query, {},
          "Filename containing bazel query results for some set of targets");
ABSL_FLAG(std::string, external_http_archive_query, "",
          "Filename containing bazel query results for external http archives");

struct ExternalProtoLibrary {
  std::string destination;
  std::string proto_prefix;
};

struct BazelRule {
  std::string clazz;
  std::string name;
  std::vector<std::string> srcs;
  std::vector<std::string> hdrs;
  std::vector<std::string> textual_hdrs;
  std::vector<std::string> deps;
  std::vector<std::string> data;
  std::vector<std::string> tags;
  std::vector<std::string> args;
  std::optional<std::string> generator_function;
  std::optional<std::string> size;
  bool flaky = false;
  std::optional<std::string> actual;  // the real target name for aliases

  bool transitive_deps_computed = false;
  std::set<std::string> transitive_deps;
  std::set<std::string> collapsed_deps;
  std::set<std::string> exclude_deps;
  std::set<std::string> collapsed_srcs;
  std::set<std::string> collapsed_public_headers;
  std::set<std::string> collapsed_headers;

  template <typename Sink>
  friend void AbslStringify(Sink& sink, const BazelRule& bazel_rule) {
    absl::Format(&sink,
                 "%s(name=%s, srcs=[%s], hdrs=[%s], textual_hdrs=[%s], "
                 "deps=[%s], data=[%s], tags=[%s], args=[%s], "
                 "generator_function=%s, size=%s, flaky=%s, actual=%s)",
                 bazel_rule.clazz, bazel_rule.name,
                 absl::StrJoin(bazel_rule.srcs, ","),
                 absl::StrJoin(bazel_rule.hdrs, ","),
                 absl::StrJoin(bazel_rule.textual_hdrs, ","),
                 absl::StrJoin(bazel_rule.deps, ","),
                 absl::StrJoin(bazel_rule.data, ","),
                 absl::StrJoin(bazel_rule.tags, ","),
                 absl::StrJoin(bazel_rule.args, ","),
                 bazel_rule.generator_function.has_value()
                     ? *bazel_rule.generator_function
                     : "None",
                 bazel_rule.size.has_value() ? *bazel_rule.size : "None",
                 bazel_rule.flaky ? "true" : "true",
                 bazel_rule.actual.has_value() ? *bazel_rule.actual : "None");
  }
};

struct HttpArchive {
  std::string name;
  std::vector<std::string> urls;
  std::string sha256;
  std::string strip_prefix;
};

BazelRule BazelRuleFromXml(const pugi::xml_node& node) {
  BazelRule out;
  out.clazz = node.attribute("class").as_string();
  out.name = node.attribute("name").as_string();
  for (const auto& child : node.children("list")) {
    auto match = [&child](absl::string_view name,
                          std::vector<std::string>* out) {
      if (child.attribute("name").as_string() != name) return;
      for (const auto& label : child.children("label")) {
        out->push_back(label.attribute("value").as_string());
      }
    };
    match("srcs", &out.srcs);
    match("hdrs", &out.hdrs);
    match("textual_hdrs", &out.textual_hdrs);
    match("deps", &out.deps);
    match("data", &out.data);
    match("tags", &out.tags);
    match("args", &out.args);
  }
  for (const auto& child : node.children("string")) {
    auto match = [&child](absl::string_view name,
                          std::optional<std::string>* out) {
      if (child.attribute("name").as_string() != name) return;
      out->emplace(child.attribute("value").as_string());
    };
    match("generator_function", &out.generator_function);
    match("size", &out.size);
  }
  for (const auto& child : node.children("boolean")) {
    auto match = [&child](absl::string_view name, bool* out) {
      if (child.attribute("name").as_string() != name) return;
      *out = 0 == strcmp(child.attribute("value").as_string(), "true");
    };
    match("flaky", &out.flaky);
  }
  for (const auto& child : node.children("label")) {
    // extract actual name for alias and bind rules
    if (0 == strcmp(child.attribute("name").as_string(), "actual")) {
      out.actual = child.attribute("value").as_string();
      out.deps.push_back(*out.actual);
    }
  }
  return out;
}

// extra metadata that will be used to construct build.yaml
// there are mostly extra properties that we weren't able to obtain from the
// bazel build _TYPE: whether this is library, target or test _RENAME: whether
// this target should be renamed to a different name (to match expectations of
// make and cmake builds)
static const char* kBuildExtraMetadata = R"json({
    "third_party/address_sorting:address_sorting": {
        "language": "c",
        "build": "all",
        "_RENAME": "address_sorting"
    },
    "@com_google_protobuf//upb/base": {
        "language": "c",
        "build": "all",
        "_RENAME": "upb_base_lib"
    },
    "@com_google_protobuf//upb/mem": {
        "language": "c",
        "build": "all",
        "_RENAME": "upb_mem_lib"
    },
    "@com_google_protobuf//upb/lex:lex": {
        "language": "c",
        "build": "all",
        "_RENAME": "upb_lex_lib"
    },
    "@com_google_protobuf//upb/message": {
        "language": "c",
        "build": "all",
        "_RENAME": "upb_message_lib"
    },
    "@com_google_protobuf//upb/json:json": {
        "language": "c",
        "build": "all",
        "_RENAME": "upb_json_lib"
    },
    "@com_google_protobuf//upb/mini_descriptor:mini_descriptor": {
        "language": "c",
        "build": "all",
        "_RENAME": "upb_mini_descriptor_lib"
    },
    "@com_google_protobuf//upb/text:text": {
        "language": "c",
        "build": "all",
        "_RENAME": "upb_textformat_lib"
    },
    "@com_google_protobuf//upb/wire:wire": {
        "language": "c",
        "build": "all",
        "_RENAME": "upb_wire_lib"
    },
    "@com_google_protobuf//third_party/utf8_range:utf8_range": {
        "language": "c",
        "build": "all",
        // rename to utf8_range_lib is necessary for now to avoid clash with utf8_range target in protobuf's cmake
        "_RENAME": "utf8_range_lib"
    },
    "@com_googlesource_code_re2//:re2": {
        "language": "c",
        "build": "all",
        "_RENAME": "re2"
    },
    "@com_google_googletest//:gtest": {
        "language": "c",
        "build": "private",
        "_RENAME": "gtest"
    },
    "@zlib//:zlib": {
        "language": "c",
        "zlib": true,
        "build": "private",
        "defaults": "zlib",
        "_RENAME": "z"
    },
    "gpr": {
        "language": "c",
        "build": "all"
    },
    "grpc": {
        "language": "c",
        "build": "all",
        "baselib": true,
        "generate_plugin_registry": true
    },
    "grpc++": {
        "language": "c++",
        "build": "all",
        "baselib": true
    },
    "grpc++_alts": {"language": "c++", "build": "all", "baselib": true},
    "grpc++_error_details": {"language": "c++", "build": "all"},
    "grpc++_reflection": {"language": "c++", "build": "all"},
    "grpc_authorization_provider": {"language": "c++", "build": "all"},
    "grpc++_unsecure": {
        "language": "c++",
        "build": "all",
        "baselib": true
    },
    "grpc_unsecure": {
        "language": "c",
        "build": "all",
        "baselib": true,
        "generate_plugin_registry": true
    },
    "grpcpp_channelz": {"language": "c++", "build": "all"},
    "grpcpp_otel_plugin": {
        "language": "c++",
        "build": "plugin"
    },
    "grpc++_test": {
        "language": "c++",
        "build": "private"
    },
    "src/compiler:grpc_plugin_support": {
        "language": "c++",
        "build": "protoc",
        "_RENAME": "grpc_plugin_support"
    },
    "src/compiler:grpc_cpp_plugin": {
        "language": "c++",
        "build": "protoc",
        "_TYPE": "target",
        "_RENAME": "grpc_cpp_plugin"
    },
    "src/compiler:grpc_csharp_plugin": {
        "language": "c++",
        "build": "protoc",
        "_TYPE": "target",
        "_RENAME": "grpc_csharp_plugin"
    },
    "src/compiler:grpc_node_plugin": {
        "language": "c++",
        "build": "protoc",
        "_TYPE": "target",
        "_RENAME": "grpc_node_plugin"
    },
    "src/compiler:grpc_objective_c_plugin": {
        "language": "c++",
        "build": "protoc",
        "_TYPE": "target",
        "_RENAME": "grpc_objective_c_plugin"
    },
    "src/compiler:grpc_php_plugin": {
        "language": "c++",
        "build": "protoc",
        "_TYPE": "target",
        "_RENAME": "grpc_php_plugin"
    },
    "src/compiler:grpc_python_plugin": {
        "language": "c++",
        "build": "protoc",
        "_TYPE": "target",
        "_RENAME": "grpc_python_plugin"
    },
    "src/compiler:grpc_ruby_plugin": {
        "language": "c++",
        "build": "protoc",
        "_TYPE": "target",
        "_RENAME": "grpc_ruby_plugin"
    },
    // TODO(jtattermusch): consider adding grpc++_core_stats
    // test support libraries
    "test/core/test_util:grpc_test_util": {
        "language": "c",
        "build": "private",
        "_RENAME": "grpc_test_util"
    },
    "test/core/test_util:grpc_test_util_unsecure": {
        "language": "c",
        "build": "private",
        "_RENAME": "grpc_test_util_unsecure"
    },
    // TODO(jtattermusch): consider adding grpc++_test_util_unsecure - it doesn't seem to be used by bazel build (don't forget to set secure: true)
    "test/cpp/util:test_config": {
        "language": "c++",
        "build": "private",
        "_RENAME": "grpc++_test_config"
    },
    "test/cpp/util:test_util": {
        "language": "c++",
        "build": "private",
        "_RENAME": "grpc++_test_util"
    },
    // benchmark support libraries
    "test/cpp/microbenchmarks:helpers": {
        "language": "c++",
        "build": "test",
        "defaults": "benchmark",
        "_RENAME": "benchmark_helpers"
    },
    "test/cpp/interop:interop_client": {
        "language": "c++",
        "build": "test",
        "run": true,
        "_TYPE": "target",
        "_RENAME": "interop_client"
    },
    "test/cpp/interop:interop_server": {
        "language": "c++",
        "build": "test",
        "run": true,
        "_TYPE": "target",
        "_RENAME": "interop_server"
    },
    // TODO(stanleycheung): re-enable this after cmake support for otel is added
    // "test/cpp/interop:xds_interop_client": {
    //     "language": "c++",
    //     "build": "test",
    //     "run": true,
    //     "_TYPE": "target",
    //     "_RENAME": "xds_interop_client",
    // },
    // "test/cpp/interop:xds_interop_server": {
    //     "language": "c++",
    //     "build": "test",
    //     "run": true,
    //     "_TYPE": "target",
    //     "_RENAME": "xds_interop_server",
    // },
    "test/cpp/interop:http2_client": {
        "language": "c++",
        "build": "test",
        "run": true,
        "_TYPE": "target",
        "_RENAME": "http2_client"
    },
    "test/cpp/qps:qps_json_driver": {
        "language": "c++",
        "build": "test",
        "run": true,
        "_TYPE": "target",
        "_RENAME": "qps_json_driver"
    },
    "test/cpp/qps:qps_worker": {
        "language": "c++",
        "build": "test",
        "run": true,
        "_TYPE": "target",
        "_RENAME": "qps_worker"
    },
    "test/cpp/util:grpc_cli": {
        "language": "c++",
        "build": "test",
        "run": true,
        "_TYPE": "target",
        "_RENAME": "grpc_cli"
    }
    // TODO(jtattermusch): create_jwt and verify_jwt breaks distribtests because it depends on grpc_test_utils and thus requires tests to be built
    // For now it's ok to disable them as these binaries aren't very useful anyway.
    // 'test/core/security:create_jwt': { 'language': 'c', 'build': 'tool', '_TYPE': 'target', '_RENAME': 'grpc_create_jwt' },
    // 'test/core/security:verify_jwt': { 'language': 'c', 'build': 'tool', '_TYPE': 'target', '_RENAME': 'grpc_verify_jwt' },
    // TODO(jtattermusch): add remaining tools such as grpc_print_google_default_creds_token (they are not used by bazel build)
    // TODO(jtattermusch): these fuzzers had no build.yaml equivalent
    // test/core/compression:message_compress_fuzzer
    // test/core/compression:message_decompress_fuzzer
    // test/core/compression:stream_compression_fuzzer
    // test/core/compression:stream_decompression_fuzzer
    // test/core/slice:b64_decode_fuzzer
    // test/core/slice:b64_encode_fuzzer
})json";

class ArtifactGen {
 public:
  void LoadRulesXml(const std::string& source) {
    pugi::xml_document doc;
    pugi::xml_parse_result result = doc.load_file(source.c_str());
    CHECK(result) << source;

    for (const auto& query : doc.children("query")) {
      for (const auto& child : query.children("rule")) {
        auto bazel_rule = BazelRuleFromXml(child);
        if (bazel_rule.clazz == "cc_library" ||
            bazel_rule.clazz == "cc_binary" || bazel_rule.clazz == "cc_test" ||
            bazel_rule.clazz == "cc_proto_library" ||
            bazel_rule.clazz == "cc_proto_gen_validate" ||
            bazel_rule.clazz == "proto_library" ||
            bazel_rule.clazz == "upb_c_proto_library" ||
            bazel_rule.clazz == "upb_proto_reflection_library" ||
            bazel_rule.clazz == "alias" || bazel_rule.clazz == "bind" ||
            bazel_rule.clazz == "genrule") {
          rules_[bazel_rule.name] = bazel_rule;
        }
      }
    }
  }

  void ExpandUpbProtoLibraryRules() {
    const std::string kGenUpbRoot = "//:src/core/ext/upb-gen/";
    const std::string kGenUpbdefsRoot = "//:src/core/ext/upbdefs-gen/";
    const std::map<std::string, std::string> kRepoNameMapping{
        {"@@cel-spec+", "@dev_cel"},
        {"@@googleapis+", "@com_google_googleapis"},
        {"@@xds+", "@com_github_cncf_xds"},
        {"@@protoc-gen-validate+", "@com_envoyproxy_protoc_gen_validate"},
        {"@@opencensus-proto+", "opencensus_proto"},
        {"@@envoy_api+", "@envoy_api"},
    };
    const std::map<std::string, std::string> kExternalLinks{
        {"@com_google_protobuf//", "src/"},
        {"@com_google_googleapis//", ""},
        {"@com_github_cncf_xds//", ""},
        {"@com_envoyproxy_protoc_gen_validate//", ""},
        {"@dev_cel//", "proto/"},
        {"@envoy_api//", ""},
        {"@opencensus_proto//", ""},
    };
    for (auto& [name, bazel_rule] : rules_) {
      if (bazel_rule.generator_function != "grpc_upb_proto_library" &&
          bazel_rule.generator_function !=
              "grpc_upb_proto_reflection_library") {
        continue;
      }
      CHECK_EQ(bazel_rule.deps.size(), 1u) << bazel_rule;
      const std::string original_dep = bazel_rule.deps[0];
      // deps is not properly fetched from bazel query for upb_c_proto_library
      // target so add the upb dependency manually
      bazel_rule.deps = {
          "@com_google_protobuf//upb:descriptor_upb_proto",
          "@com_google_protobuf//"
          "upb:generated_code_support"
          "give_permission_to_break_me",
      };
      // populate the upb_c_proto_library rule with pre-generated upb headers
      // and sources using proto_rule
      auto get_prefix_to_strip =
          [&kExternalLinks, &kRepoNameMapping,
           &name](const std::string& proto_src) -> std::string {
        for (const auto& [prefix, expected_dir] : kExternalLinks) {
          if (absl::StartsWith(proto_src, prefix)) {
            std::string prefix_to_strip = prefix + expected_dir;
            CHECK(absl::StartsWith(proto_src, prefix_to_strip))
                << "Source file " << proto_src << " in upb rule " << name
                << " does not have the expected prefix " << prefix_to_strip;
            return prefix_to_strip;
          }
        }
        for (const auto& [canonical_repo, apparent_repo] : kRepoNameMapping) {
          if (absl::StartsWith(proto_src, canonical_repo)) {
            const std::string apparent_repo =
                kRepoNameMapping.at(canonical_repo) + "//";
            std::string prefix_to_strip =
                canonical_repo + "//" + kExternalLinks.at(apparent_repo);
            CHECK(absl::StartsWith(proto_src, prefix_to_strip))
                << "Source file " << proto_src << " in upb rule " << name
                << " does not have the expected prefix " << prefix_to_strip;
            return prefix_to_strip;
          }
        }
        return "";
      };
      const auto protos = GetTransitiveProtos(original_dep);
      CHECK_NE(protos.size(), 0u);
      std::vector<std::string> files;
      for (std::string proto_src : protos) {
        const std::string prefix_to_strip = get_prefix_to_strip(proto_src);
        // A no-op if this isn't a proto target.
        proto_src = proto_src.substr(prefix_to_strip.length());
        CHECK(!absl::StartsWith(proto_src, "@"))
            << name << " is unknown workspace; proto_src=" << proto_src;
        const std::string proto_src_file =
            TryExtractSourceFilePath(proto_src).value();
        std::vector<std::string> extensions;
        std::string root;
        if (bazel_rule.generator_function == "grpc_upb_proto_library") {
          extensions = {".upb.h", ".upb_minitable.h", ".upb_minitable.c"};
          root = kGenUpbRoot;
        } else {
          extensions = {".upbdefs.h", ".upbdefs.c"};
          root = kGenUpbdefsRoot;
        }
        for (const auto& ext : extensions) {
          files.push_back(absl::StrCat(
              root, absl::StrReplaceAll(proto_src_file, {{".proto", ext}})));
        }
      }
      bazel_rule.srcs = files;
      bazel_rule.hdrs = files;
    }
  }

  void PatchGrpcProtoLibraryRules() {
    for (auto& [name, bazel_rule] : rules_) {
      if (absl::StartsWith(name, "//") &&
          (bazel_rule.generator_function == "grpc_proto_library" ||
           bazel_rule.clazz == "cc_proto_library")) {
        bazel_rule.deps.push_back("//third_party:protobuf");
      }
    }
  }

  void PatchDescriptorUpbProtoLibrary() {
    auto it = rules_.find("@com_google_protobuf//upb:descriptor_upb_proto");
    if (it == rules_.end()) return;
    auto& bazel_rule = it->second;
    bazel_rule.srcs.push_back(
        ":src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c");
    bazel_rule.hdrs.push_back(
        ":src/core/ext/upb-gen/google/protobuf/descriptor.upb.h");
  }

  void PopulateCcTests() {
    for (const auto& [_, bazel_rule] : rules_) {
      if (bazel_rule.clazz != "cc_test") continue;
      std::string test_name = bazel_rule.name;
      if (!absl::StartsWith(test_name, "//")) continue;
      test_name = test_name.substr(2);
      if (!WantCcTest(test_name)) continue;
      tests_.push_back(test_name);
    }
  }

  void GenerateBuildExtraMetadataForTests() {
    for (const auto& test : tests_) {
      nlohmann::json test_dict = nlohmann::json::object();
      test_dict["build"] = "test";
      test_dict["_TYPE"] = "target";
      auto bazel_rule = *LookupRule(test);
      if (absl::c_contains(bazel_rule.tags, "manual")) {
        test_dict["run"] = false;
      }
      if (bazel_rule.flaky) {
        test_dict["run"] = false;
      }
      if (absl::c_contains(bazel_rule.tags, "no_uses_polling")) {
        test_dict["uses_polling"] = false;
      }
      if (absl::c_contains(bazel_rule.tags, "bazel_only")) {
        continue;
      }
      if (absl::StartsWith(test, "test/cpp/ext/otel")) {
        test_dict["build"] = "plugin_test";
        test_dict["plugin_option"] = "gRPC_BUILD_GRPCPP_OTEL_PLUGIN";
      }
      // if any tags that restrict platform compatibility are present,
      // generate the "platforms" field accordingly
      // TODO(jtattermusch): there is also a "no_linux" tag, but we cannot
      // take it into account as it is applied by grpc_cc_test when poller
      // expansion is made (for tests where uses_polling=true). So for now, we
      // just assume all tests are compatible with linux and ignore the
      // "no_linux" tag completely.
      if (absl::c_contains(bazel_rule.tags, "no_windows") ||
          absl::c_contains(bazel_rule.tags, "no_mac")) {
        nlohmann::json platforms = nlohmann::json::array();
        platforms.push_back("linux");
        platforms.push_back("posix");
        if (!absl::c_contains(bazel_rule.tags, "no_windows")) {
          platforms.push_back("windows");
        }
        if (!absl::c_contains(bazel_rule.tags, "no_mac")) {
          platforms.push_back("mac");
        }
        test_dict["platforms"] = platforms;
      }
      if (!bazel_rule.args.empty()) {
        test_dict["args"] = bazel_rule.args;
      }
      if (absl::StartsWith(test, "test/cpp")) {
        test_dict["language"] = "c++";
      } else if (absl::StartsWith(test, "test/core")) {
        test_dict["language"] = "c";
      } else {
        LOG(FATAL) << "wrong test: " << test;
      }
      test_metadata_[test] = test_dict;
    }
    const auto extra =
        nlohmann::json::parse(kBuildExtraMetadata, nullptr, true, true);
    for (auto it = extra.begin(); it != extra.end(); ++it) {
      test_metadata_[it.key()] = it.value();
    }
  }

  void PopulateTransitiveMetadata() {
    for (auto it = test_metadata_.begin(); it != test_metadata_.end(); ++it) {
      bazel_label_to_dep_name_[GetBazelLabel(it.key())] = it.key();
    }
    for (auto& [rule_name, rule] : rules_) {
      if (rule.transitive_deps_computed) continue;
      ComputeTransitiveMetadata(rule);
    }
  }

  void UpdateTestMetadataWithTransitiveMetadata() {
    for (auto it = test_metadata_.begin(); it != test_metadata_.end(); ++it) {
      const auto& lib_name = it.key();
      auto& lib_dict = it.value();
      if (lib_dict["build"] != "test" && lib_dict["build"] != "plugin_test") {
        continue;
      }
      if (!lib_dict.contains("_TYPE") || lib_dict["_TYPE"] != "target") {
        continue;
      }
      const auto& bazel_rule = *LookupRule(lib_name);
      if (bazel_rule.transitive_deps.count("//third_party:benchmark") > 0) {
        lib_dict["benchmark"] = true;
        lib_dict["defaults"] = "benchmark";
      }
      if (bazel_rule.transitive_deps.count("//third_party:gtest") > 0) {
        lib_dict["gtest"] = true;
        lib_dict["language"] = "c++";
      }
    }
  }

  void GenerateBuildMetadata() {
    std::vector<std::string> lib_names;
    for (auto it = test_metadata_.begin(); it != test_metadata_.end(); ++it) {
      lib_names.push_back(it.key());
    }
    for (const auto& lib_name : lib_names) {
      auto lib_dict = CreateTargetFromBazelRule(lib_name);
      lib_dict.update(test_metadata_[lib_name]);
      build_metadata_[lib_name] = lib_dict;
    }
  }

  void ConvertToBuildYamlLike() {
    std::vector<nlohmann::json> lib_list;
    std::vector<nlohmann::json> target_list;
    std::vector<nlohmann::json> test_list;
    for (auto it = build_metadata_.begin(); it != build_metadata_.end(); ++it) {
      const auto& lib_dict = it.value();
      if (!lib_dict.contains("_TYPE")) {
        lib_list.push_back(lib_dict);
      } else if (lib_dict["_TYPE"] == "library") {
        lib_list.push_back(lib_dict);
      } else if (lib_dict["_TYPE"] == "target") {
        target_list.push_back(lib_dict);
      } else if (lib_dict["_TYPE"] == "test" ||
                 lib_dict["_TYPE"] == "plugin_test") {
        test_list.push_back(lib_dict);
      }
    }
    auto scrub = [](nlohmann::json& lib,
                    std::initializer_list<std::string> explicit_fields) {
      std::vector<std::string> fields_to_remove = explicit_fields;
      for (auto it = lib.begin(); it != lib.end(); ++it) {
        if (absl::StartsWith(it.key(), "_")) {
          fields_to_remove.push_back(it.key());
        }
      }
      for (const auto& field : fields_to_remove) {
        lib.erase(field);
      }
    };
    for (auto& lib : lib_list) {
      scrub(lib, {});
    }
    for (auto& target : target_list) {
      scrub(target, {"public_headers"});
    }
    for (auto& test : test_list) {
      scrub(test, {"public_headers"});
    }
    build_yaml_like_ = {
        {"libs", lib_list},
        {"filegroups", {}},
        {"targets", target_list},
        {"tests", test_list},
    };
  }

  void GenerateExternalProtoLibraries() {
    pugi::xml_document doc;
    std::string filename = absl::GetFlag(FLAGS_external_http_archive_query);
    CHECK(!filename.empty()) << "external_http_archive_query is not set";

    std::vector<nlohmann::json> external_proto_libraries;
    std::ifstream fs(filename);
    nlohmann::json http_archives = nlohmann::json::parse(fs);
    // Make a copy for each element
    for (nlohmann::json http_archive: http_archives["http_archives"]){
        const std::string name = http_archive["name"].get<std::string>();
        if (external_proto_libraries_.count(name) == 0) {
          // If this http archive is not one of the external proto libraries,
          // we don't want to include it as a CMake target
          continue;
        }
        const auto& extlib =
            external_proto_libraries_.find(name)->second;
        http_archive["destination"] = extlib.destination;
        http_archive["proto_prefix"] = extlib.proto_prefix;
        external_proto_libraries.push_back(http_archive);
    }

    build_yaml_like_["external_proto_libraries"] = external_proto_libraries;
  }

  nlohmann::json Result() { return build_yaml_like_; }

 private:
  // Computes the final build metadata for Bazel target with rule_name.
  //
  // The dependencies that will appear on the deps list are:
  //
  // * Public build targets including binaries and tests;
  // * External targets, like absl, re2.
  //
  // All other intermediate dependencies will be merged, which means their
  // source file, headers, etc. will be collected into one build target. This
  // step of processing will greatly reduce the complexity of the generated
  // build specifications for other build systems, like CMake, Make,
  // setuptools.
  //
  // The final build metadata are:
  // * _TRANSITIVE_DEPS: all the transitive dependencies including
  // intermediate
  //                     targets;
  // * _COLLAPSED_DEPS:  dependencies that fits our requirement above, and it
  //                     will remove duplicated items and produce the shortest
  //                     possible dependency list in alphabetical order;
  // * _COLLAPSED_SRCS:  the merged source files;
  // * _COLLAPSED_PUBLIC_HEADERS: the merged public headers;
  // * _COLLAPSED_HEADERS: the merged non-public headers;
  // * _EXCLUDE_DEPS: intermediate targets to exclude when performing
  // collapsing
  //      of sources and dependencies.
  //
  // For the collapsed_deps, the algorithm improved cases like:
  //
  // The result in the past:
  //     end2end_tests -> [grpc_test_util, grpc, gpr, address_sorting, upb]
  //     grpc_test_util -> [grpc, gpr, address_sorting, upb, ...]
  //     grpc -> [gpr, address_sorting, upb, ...]
  //
  // The result of the algorithm:
  //     end2end_tests -> [grpc_test_util]
  //     grpc_test_util -> [grpc]
  //     grpc -> [gpr, address_sorting, upb, ...]
  void ComputeTransitiveMetadata(BazelRule& bazel_rule) {
    auto direct_deps = ExtractDeps(bazel_rule);
    std::set<std::string> transitive_deps;
    std::set<std::string> collapsed_deps;
    std::set<std::string> exclude_deps;
    std::set<std::string> collapsed_srcs = ExtractSources(bazel_rule);
    std::set<std::string> collapsed_public_headers =
        ExtractPublicHeaders(bazel_rule);
    std::set<std::string> collapsed_headers =
        ExtractNonPublicHeaders(bazel_rule);

    auto update = [](const std::set<std::string>& add,
                     std::set<std::string>& to) {
      for (const auto& a : add) to.insert(a);
    };

    for (const auto& dep : direct_deps) {
      auto external_dep_name_maybe = ExternalDepNameFromBazelDependency(dep);

      auto it = rules_.find(dep);
      if (it != rules_.end()) {
        BazelRule& dep_rule = it->second;
        // Descend recursively, but no need to do that for external deps
        if (!external_dep_name_maybe.has_value()) {
          if (!dep_rule.transitive_deps_computed) {
            ComputeTransitiveMetadata(dep_rule);
          }
          update(dep_rule.transitive_deps, transitive_deps);
          update(dep_rule.collapsed_deps, collapsed_deps);
          update(dep_rule.exclude_deps, exclude_deps);
        }
      }
      // This dep is a public target, add it as a dependency
      auto it_bzl = bazel_label_to_dep_name_.find(dep);
      if (it_bzl != bazel_label_to_dep_name_.end()) {
        transitive_deps.insert(it_bzl->second);
        collapsed_deps.insert(it_bzl->second);
        // Add all the transitive deps of our every public dep to exclude
        // list since we want to avoid building sources that are already
        // built by our dependencies
        update(rules_[dep].transitive_deps, exclude_deps);
        continue;
      }
      // This dep is an external target, add it as a dependency
      if (external_dep_name_maybe.has_value()) {
        transitive_deps.insert(external_dep_name_maybe.value());
        collapsed_deps.insert(external_dep_name_maybe.value());
        continue;
      }
    }
    // Direct dependencies are part of transitive dependencies
    update(direct_deps, transitive_deps);
    // Calculate transitive public deps (needed for collapsing sources)
    std::set<std::string> transitive_public_deps;
    for (const auto& dep : transitive_deps) {
      if (bazel_label_to_dep_name_.count(dep) > 0) {
        transitive_public_deps.insert(dep);
      }
    }
    // Remove intermediate targets that our public dependencies already depend
    // on. This is the step that further shorten the deps list.
    std::set<std::string> new_collapsed_deps;
    for (const auto& dep : collapsed_deps) {
      if (exclude_deps.count(dep) == 0) {
        new_collapsed_deps.insert(dep);
      }
    }
    collapsed_deps.swap(new_collapsed_deps);
    // Compute the final source files and headers for this build target whose
    // name is `rule_name` (input argument of this function).
    //
    // Imaging a public target PX has transitive deps [IA, IB, PY, IC, PZ].
    // PX, PY and PZ are public build targets. And IA, IB, IC are intermediate
    // targets. In addition, PY depends on IC.
    //
    // Translate the condition into dependency graph:
    //   PX -> [IA, IB, PY, IC, PZ]
    //   PY -> [IC]
    //   Public targets: [PX, PY, PZ]
    //
    // The collapsed dependencies of PX: [PY, PZ].
    // The excluded dependencies of X: [PY, IC, PZ].
    // (IC is excluded as a dependency of PX. It is already included in PY,
    // hence it would be redundant to include it again.)
    //
    // Target PX should include source files and headers of [PX, IA, IB] as
    // final build metadata.
    for (const auto& dep : transitive_deps) {
      if (exclude_deps.count(dep) == 0 &&
          transitive_public_deps.count(dep) == 0) {
        if (rules_.count(dep) != 0) {
          update(ExtractSources(rules_[dep]), collapsed_srcs);
          update(ExtractPublicHeaders(rules_[dep]), collapsed_public_headers);
          update(ExtractNonPublicHeaders(rules_[dep]), collapsed_headers);
        }
      }
    }
    bazel_rule.transitive_deps_computed = true;
    bazel_rule.transitive_deps = std::move(transitive_deps);
    bazel_rule.collapsed_deps = std::move(collapsed_deps);
    bazel_rule.exclude_deps = std::move(exclude_deps);
    bazel_rule.collapsed_srcs = std::move(collapsed_srcs);
    bazel_rule.collapsed_public_headers = std::move(collapsed_public_headers);
    bazel_rule.collapsed_headers = std::move(collapsed_headers);
  }

  // Returns name of dependency if external bazel dependency is provided or
  // nullopt
  std::optional<std::string> ExternalDepNameFromBazelDependency(
      std::string bazel_dep) {
    if (absl::StartsWith(bazel_dep, "@com_google_absl//")) {
      return bazel_dep.substr(strlen("@com_google_absl//"));
    }
    if (bazel_dep == "@com_github_google_benchmark//:benchmark") {
      return "benchmark";
    }
    if (bazel_dep == "@boringssl//:ssl") {
      return "libssl";
    }
    if (bazel_dep == "@com_github_cares_cares//:ares") {
      return "cares";
    }
    if (bazel_dep == "@com_google_protobuf//:protobuf" ||
        bazel_dep == "@com_google_protobuf//:protobuf_headers") {
      return "protobuf";
    }
    if (bazel_dep == "@com_google_protobuf//:protoc_lib") {
      return "protoc";
    }
    if (bazel_dep == "@io_opentelemetry_cpp//api:api") {
      return "opentelemetry-cpp::api";
    }
    if (bazel_dep == "@io_opentelemetry_cpp//sdk/src/metrics:metrics") {
      return "opentelemetry-cpp::metrics";
    }
    // Two options here:
    // * either this is not external dependency at all (which is fine, we will
    //   treat it as internal library)
    // * this is external dependency, but we don't want to make the dependency
    //   explicit in the build metadata for other build systems.
    return std::nullopt;
  }

  std::set<std::string> ExtractDeps(const BazelRule& bazel_rule) {
    std::set<std::string> deps;
    for (const auto& dep : bazel_rule.deps) {
      deps.insert(dep);
    }
    for (const auto& src : bazel_rule.srcs) {
      if (!absl::EndsWith(src, ".cc") && !absl::EndsWith(src, ".c") &&
          !absl::EndsWith(src, ".proto")) {
        auto rule_it = rules_.find(src);
        if (rule_it != rules_.end()) {
          // This label doesn't point to a source file, but another Bazel
          // target. This is required for :pkg_cc_proto_validate targets,
          // and it's generally allowed by Bazel.
          deps.insert(src);
        }
      }
    }
    return deps;
  }

  std::set<std::string> ExtractSources(const BazelRule& bazel_rule) {
    std::set<std::string> srcs;
    for (const auto& src : bazel_rule.srcs) {
      if (absl::StartsWith(src, "@com_google_protobuf//") &&
          absl::EndsWith(src, ".proto")) {
        continue;
      }
      if (absl::EndsWith(src, ".cc") || absl::EndsWith(src, ".c") ||
          absl::EndsWith(src, ".proto")) {
        auto source_file_maybe = TryExtractSourceFilePath(src);
        if (source_file_maybe.has_value()) {
          srcs.insert(source_file_maybe.value());
        }
      }
    }
    return srcs;
  }

  std::set<std::string> ExtractPublicHeaders(const BazelRule& bazel_rule) {
    std::set<std::string> headers;
    for (const auto& hdr : bazel_rule.hdrs) {
      if (absl::StartsWith(hdr, "//:include/") && HasHeaderSuffix(hdr)) {
        auto source_file_maybe = TryExtractSourceFilePath(hdr);
        if (source_file_maybe.has_value()) {
          headers.insert(source_file_maybe.value());
        }
      }
    }
    return headers;
  }

  std::set<std::string> ExtractNonPublicHeaders(const BazelRule& bazel_rule) {
    std::set<std::string> headers;
    for (const auto* vec :
         {&bazel_rule.hdrs, &bazel_rule.textual_hdrs, &bazel_rule.srcs}) {
      for (const auto& hdr : *vec) {
        if (!absl::StartsWith(hdr, "//:include/") && HasHeaderSuffix(hdr)) {
          auto source_file_maybe = TryExtractSourceFilePath(hdr);
          if (source_file_maybe.has_value()) {
            headers.insert(source_file_maybe.value());
          }
        }
      }
    }
    return headers;
  }

  bool HasHeaderSuffix(absl::string_view hdr) {
    return absl::EndsWith(hdr, ".h") || absl::EndsWith(hdr, ".hpp") ||
           absl::EndsWith(hdr, ".inc");
  }

  bool WantCcTest(absl::string_view test) {
    // most qps tests are autogenerated, we are fine without them
    if (absl::StartsWith(test, "test/cpp/qps:")) return false;
    // microbenchmarks aren't needed for checking correctness
    if (absl::StartsWith(test, "test/cpp/microbenchmarks:")) return false;
    if (absl::StartsWith(test, "test/core/promise/benchmark:")) return false;
    // we have trouble with census dependency outside of bazel
    if (absl::StartsWith(test, "test/cpp/ext/filters/census:")) return false;
    if (absl::StartsWith(test,
                         "test/core/server:xds_channel_stack_modifier_test")) {
      return false;
    }
    if (absl::StartsWith(test, "test/cpp/ext/gcp:")) return false;
    if (absl::StartsWith(test, "test/cpp/ext/filters/logging:")) return false;
    if (absl::StartsWith(test, "test/cpp/interop:observability_interop")) {
      return false;
    }
    // we have not added otel dependency outside of bazel
    if (absl::StartsWith(test, "test/cpp/ext/csm:")) return false;
    if (absl::StartsWith(test, "test/cpp/interop:xds_interop")) return false;
    // missing opencensus/stats/stats.h
    if (absl::StartsWith(
            test, "test/cpp/end2end:server_load_reporting_end2end_test")) {
      return false;
    }
    if (absl::StartsWith(
            test, "test/cpp/server/load_reporter:lb_load_reporter_test")) {
      return false;
    }
    // The test uses --running_under_bazel cmdline argument
    // To avoid the trouble needing to adjust it, we just skip the test
    if (absl::StartsWith(
            test, "test/cpp/naming:resolver_component_tests_runner_invoker")) {
      return false;
    }
    // the test requires 'client_crash_test_server' to be built
    if (absl::StartsWith(test, "test/cpp/end2end:time_change_test")) {
      return false;
    }
    if (absl::StartsWith(test, "test/cpp/end2end:client_crash_test")) {
      return false;
    }
    // the test requires 'server_crash_test_client' to be built
    if (absl::StartsWith(test, "test/cpp/end2end:server_crash_test")) {
      return false;
    }
    // test never existed under build.yaml and it fails -> skip it
    if (absl::StartsWith(test, "test/core/tsi:ssl_session_cache_test")) {
      return false;
    }
    // the binary of this test does not get built with cmake
    if (absl::StartsWith(test, "test/cpp/util:channelz_sampler_test")) {
      return false;
    }
    // chaotic good not supported outside bazel
    if (absl::StartsWith(test, "test/core/transport/chaotic_good")) {
      return false;
    }
    // we don't need to generate fuzzers outside of bazel
    if (absl::EndsWith(test, "_fuzzer")) return false;
    if (absl::StrContains(test, "_fuzzer_")) return false;
    return true;
  }

  std::set<std::string> GetTransitiveProtos(std::string root) {
    std::queue<std::string> todo;
    std::set<std::string> visited;
    std::set<std::string> ret;
    todo.push(root);
    while (!todo.empty()) {
      std::string name = todo.front();
      todo.pop();
      auto rule_it = rules_.find(name);
      if (rule_it == rules_.end()) continue;
      const auto& rule = rule_it->second;
      for (const auto& dep : rule.deps) {
        if (visited.emplace(dep).second) todo.push(dep);
      }
      for (const auto& src : rule.srcs) {
        if (absl::EndsWith(src, ".proto")) ret.insert(src);
      }
    }
    return ret;
  }

  std::optional<std::string> TryExtractSourceFilePath(std::string label) {
    if (absl::StartsWith(label, "@")) {
      // This is an external source file. We are only interested in sources
      // for some of the external libraries.
      for (const auto& [lib_name, prefix] : external_source_prefixes_) {
        if (absl::StartsWith(label, lib_name)) {
          return absl::StrReplaceAll(
              label, {{lib_name, prefix}, {":", "/"}, {"//", "/"}});
        }
      }
      // This source file is external, and we need to translate the
      // @REPO_NAME to a valid path prefix. At this stage, we need
      // to check repo name, since the label/path mapping is not
      // available in BUILD files.
      for (const auto& [lib_name, external_proto_lib] :
           external_proto_libraries_) {
        if (absl::StartsWith(label, "@" + lib_name + "//")) {
          return absl::StrReplaceAll(label, {{absl::StrCat("@", lib_name, "//"),
                                              external_proto_lib.proto_prefix},
                                             {":", "/"}});
        }
      }
      // No external library match found.
      return std::nullopt;
    }
    if (absl::StartsWith(label, "//")) label = label.substr(2);
    if (absl::StartsWith(label, ":")) label = label.substr(1);
    return absl::StrReplaceAll(label, {{":", "/"}});
  }

  static std::string GetBazelLabel(std::string target_name) {
    if (absl::StartsWith(target_name, "@")) {
      if (absl::StrContains(target_name, ':')) {
        return target_name;
      } else {
        // @foo//bar/baz -> @foo//bar/baz:baz
        std::vector<std::string> parts = absl::StrSplit(target_name, '/');
        return absl::StrCat(target_name, ":", parts.back());
      }
    }
    if (absl::StrContains(target_name, ":")) {
      return absl::StrCat("//", target_name);
    } else {
      return absl::StrCat("//:", target_name);
    }
  }

  BazelRule* LookupRule(std::string target_name) {
    auto it = rules_.find(GetBazelLabel(target_name));
    if (it == rules_.end()) {
      LOG(ERROR) << "Rule not found: " << target_name
                 << " bazel label: " << GetBazelLabel(target_name);
      return nullptr;
    }
    return &it->second;
  }

  nlohmann::json CreateTargetFromBazelRule(std::string target_name) {
    auto bazel_rule = *LookupRule(target_name);
    return nlohmann::json{
        {"name", target_name},
        {"_PUBLIC_HEADERS_BAZEL", ExtractPublicHeaders(bazel_rule)},
        {"_HEADERS_BAZEL", ExtractNonPublicHeaders(bazel_rule)},
        {"_SRC_BAZEL", ExtractSources(bazel_rule)},
        {"_DEPS_BAZEL", ExtractDeps(bazel_rule)},
        {"public_headers", bazel_rule.collapsed_public_headers},
        {"headers", bazel_rule.collapsed_headers},
        {"src", bazel_rule.collapsed_srcs},
        {"deps", bazel_rule.collapsed_deps},
        {"transitive_deps", bazel_rule.transitive_deps},
        {"exclude_deps", bazel_rule.exclude_deps},
        {"collapsed_deps", bazel_rule.collapsed_deps},
        {"collapsed_headers", bazel_rule.collapsed_headers},
        {"collapsed_srcs", bazel_rule.collapsed_srcs},
    };
  }

  std::map<std::string, BazelRule> rules_;
  std::vector<std::string> tests_;
  nlohmann::json test_metadata_ = nlohmann::json::object();
  nlohmann::json build_metadata_ = nlohmann::json::object();
  nlohmann::json build_yaml_like_ = nlohmann::json::object();
  std::map<std::string, std::string> bazel_label_to_dep_name_;

  const std::map<std::string, std::string> external_source_prefixes_ = {
      {"@com_googlesource_code_re2//", "third_party/re2"},
      {"@com_google_googletest//", "third_party/googletest"},
      {"@googletest//", "third_party/googletest"},
      {"@com_google_protobuf//upb", "third_party/upb/upb"},
      {"@com_google_protobuf//third_party/utf8_range",
       "third_party/utf8_range"},
      {
          "@zlib//",
          "third_party/zlib",
      },
  };
  const std::map<std::string, ExternalProtoLibrary> external_proto_libraries_ =
      {{"envoy_api",
        {
            "third_party/envoy-api",
            "third_party/envoy-api/",
        }},
       {"com_google_googleapis",
        {
            "third_party/googleapis",
            "third_party/googleapis/",
        }},
       {"com_github_cncf_xds", {"third_party/xds", "third_party/xds/"}},
       {"com_envoyproxy_protoc_gen_validate",
        {
            "third_party/protoc-gen-validate",
            "third_party/protoc-gen-validate/",
        }},
       {
           "opencensus_proto",
           {
               "third_party/opencensus-proto/src",
               "third_party/opencensus-proto/src/",
           },
       }};
};

nlohmann::json ExtractMetadataFromBazelXml() {
  ArtifactGen generator;
  for (auto target_query : absl::GetFlag(FLAGS_target_query)) {
    generator.LoadRulesXml(target_query);
  }
  generator.ExpandUpbProtoLibraryRules();
  generator.PatchGrpcProtoLibraryRules();
  generator.PatchDescriptorUpbProtoLibrary();
  generator.PopulateCcTests();
  generator.GenerateBuildExtraMetadataForTests();
  generator.PopulateTransitiveMetadata();
  generator.UpdateTestMetadataWithTransitiveMetadata();
  generator.GenerateBuildMetadata();
  generator.ConvertToBuildYamlLike();
  generator.GenerateExternalProtoLibraries();
  return generator.Result();
}
