Merge branch 'main' into lhk_dev

Author: ValKmjolnir

BUILD

@@ -378,28 +378,28 @@ pkg_files(
 )
 
 # Pack ArkTs artifacts
-# pkg_files(
-#     name = "coref-arkts-lib-script-pkg",
-#     srcs = [
-#         "//language/arkts:lib",
-#     ],
-#     attributes = pkg_attributes(
-#         mode = "0644",
-#     ),
-#     prefix = "lib/coref/arkts",
-#     strip_prefix = strip_prefix.from_pkg() + "lib",
-# )
+pkg_files(
+    name = "coref-arkts-lib-script-pkg",
+    srcs = [
+        "//language/arkts:lib",
+    ],
+    attributes = pkg_attributes(
+        mode = "0644",
+    ),
+    prefix = "lib/coref/arkts",
+    strip_prefix = strip_prefix.from_pkg() + "lib",
+)
 
-# pkg_files(
-#     name = "coref-arkts-src-extractor-pkg",
-#     srcs = [
-#         "//language/arkts/extractor:coref-arkts-src-extractor",
-#     ],
-#     attributes = pkg_attributes(
-#         mode = "0755",
-#     ),
-#     prefix = "language/arkts/extractor",
-# )
+pkg_files(
+    name = "coref-arkts-src-extractor-pkg",
+    srcs = [
+        "//language/arkts/extractor:coref-arkts-src-extractor",
+    ],
+    attributes = pkg_attributes(
+        mode = "0755",
+    ),
+    prefix = "language/arkts/extractor",
+)
 
 # Current pkg_tar/pkg_files rules can't pack a directory generated from genrule.
 # To workaround this, use the deprecated pkg_tar rule instead
@@ -506,7 +506,7 @@ coref_lib_script_pkgs = [
     ":coref-cfamily-lib-script-pkg",
     ":coref-properties-lib-script-pkg",
     ":coref-sql-lib-script-pkg",
-    # ":coref-arkts-lib-script-pkg",
+    ":coref-arkts-lib-script-pkg",
 ] 
 
 coref_libscript_languages = [
@@ -519,7 +519,7 @@ coref_libscript_languages = [
     "properties",
     "sql",
     # "swift",
-    # "arkts",
+    "arkts",
 ]
 
 copy_to_directory(
@@ -626,15 +626,15 @@ copy_to_directory(
 #     visibility = ["//visibility:public"],
 # )
 
-# copy_to_directory(
-#     name = "copy-arkts-lib-script",
-#     srcs = [
-#         "//language/arkts:lib",
-#     ],
-#     out = "lib/coref/arkts",
-#     replace_prefixes = {"language/arkts/lib": ""},
-#     visibility = ["//visibility:public"],
-# )
+copy_to_directory(
+    name = "copy-arkts-lib-script",
+    srcs = [
+        "//language/arkts:lib",
+    ],
+    out = "lib/coref/arkts",
+    replace_prefixes = {"language/arkts/lib": ""},
+    visibility = ["//visibility:public"],
+)
 
 genrule(
     name = "coref-lib-script-gen",
@@ -650,7 +650,7 @@ genrule(
         ":copy-sql-lib-script",
         ":copy-cfamily-lib-script",
         # ":copy-swift-lib-script",
-        # ":copy-arkts-lib-script",
+        ":copy-arkts-lib-script",
     ],
     outs = ["lib/coref/%s.gdl" % language for language in coref_libscript_languages],
     cmd = """
@@ -680,7 +680,7 @@ pkg_tar(
     name = "sparrow-cli",
     srcs = [
         ":coref-lib-pkg",
-        # ":coref-arkts-src-extractor-pkg",
+        ":coref-arkts-src-extractor-pkg",
         # ":coref-go-src-extractor-pkg",
         ":coref-sql-src-extractor-pkg",
         ":coref-java-src-extractor-pkg",

README.md

@@ -92,28 +92,28 @@ Note: The maturity level of the language status is determined based on the types
 ## Tutorial
 - [Online Tutorial](./tutorial/README.en.md)
 
-## Directory Structure Description
-- `cli`: The entry point for the command-line tool, providing a unified command-line interface, calling other modules to complete specific functions
-- `language`: Core data and data modeling (lib) for various languages. Regarding the degree of openness, please refer to the section "Some Notes on the Scope of Open Source"
-- `doc`: Reference documents
-- `examples`: Gödel query language examples
-- `tutorial`：CodeFuse-Query Development Container Usage Tutorial
-
-## Some Notes on the Scope of Open Source
-As of now, it is **not possible** to build an executable program from the source code because not all modules have been made open-source in this release, and missing modules will be released over the next year. Nevertheless, to ensure a complete experience, we have released **complete installation packages** for download, please see the Release page.
-Regarding the openness of languages, you can refer to the table below:
-
-| Language | Data Modeling Open Source | Data Core Open Source | Maturity |
-| --- | --- | --- | --- |
-| Python | Y | Y | RELEASE |
-| Java | Y | Y | RELEASE |
-| JavaScript | Y | Y | RELEASE |
-| Go | Y | Y | RELEASE |
-| XML | Y | Y | RELEASE |
-| Cfamily | Y | Y | BETA |
-| SQL | Y | Y | BETA |
-| Swift | N | N | BETA |
-| Properties | Y | Y | BETA |
+
+## Related Paper  
+
+Our work is described in the following paper:  
+
+[**CodeFuse-Query: A Data-Centric Static Code Analysis System for Large-Scale Organizations**](https://arxiv.org/abs/2401.01571)  
+
+## Citation  
+
+If you find **CodeFuse-Query** useful in your research, please cite our paper:  
+
+```bibtex
+@misc{xie2024codefusequerydatacentricstaticcode,
+      title={CodeFuse-Query: A Data-Centric Static Code Analysis System for Large-Scale Organizations}, 
+      author={Xiaoheng Xie and Gang Fan and Xiaojun Lin and Ang Zhou and Shijie Li and Xunjin Zheng and Yinan Liang and Yu Zhang and Na Yu and Haokun Li and Xinyu Chen and Yingzhuang Chen and Yi Zhen and Dejun Dong and Xianjin Fu and Jinzhou Su and Fuxiong Pan and Pengshuai Luo and Youzheng Feng and Ruoxiang Hu and Jing Fan and Jinguo Zhou and Xiao Xiao and Peng Di},
+      year={2024},
+      eprint={2401.01571},
+      archivePrefix={arXiv},
+      primaryClass={cs.SE},
+      url={https://arxiv.org/abs/2401.01571}, 
+}
+```
 
 ## Contact Us
 ![WeChat User Group Image](./assets/wechat_qrcode.JPG)

README_cn.md

@@ -84,28 +84,6 @@ CodeFuse-Query 包括**Sparrow CLI **和CodeFuse-Query**在线服务Query中心*
 ## 教程 (tutorial)
 - [在线教程](./tutorial/README.md)
 
-## 目录结构说明
-- `cli`：命令行工具的入口，提供统一的命令行接口，调用其他模块完成具体功能
-- `language`：各语言的数据化核心（extractor）和数据建模（lib）。关于开放度的问题，请参见《关于开源范围的一些说明》章节
-- `doc`：参考文档
-- `examples`：Gödel 查询语言示例
-- `tutorial`：CodeFuse-Query 开发容器使用教程
-
-## 关于开源范围的一些说明
-截止目前，从源码**不能**构建出可执行的程序，原因在于本次开源并没有开放所有的模块，缺少的模块会在之后的一年陆续开源。尽管如此，为保障完整的体验，我们开放了**完整的安装包**下载，请见Release页面。
-关于语言的开放程度，可以查看下表：
-
-| 语言 | 数据建模开源 | 数据化核心开源 | 成熟度 |
-| --- | --- | --- | --- |
-| Python | Y | Y | RELEASE |
-| Java | Y | Y | RELEASE |
-| JavaScript | Y | Y | RELEASE |
-| Go | Y | Y | RELEASE |
-| XML | Y | Y | RELEASE |
-| Cfamily | Y | Y | BETA |
-| SQL | Y | Y | BETA |
-| Swift | N | N | BETA |
-| Properties | Y | Y | BETA |
 
 ## 联系我们
 ![微信用户群图片](./assets/wechat_qrcode.JPG)

WORKSPACE

@@ -560,3 +560,20 @@ aspect_bazel_lib_dependencies()
 #     sha256 = "d4292d9b898a9d264c3e8ea58c83074a63d0c8d84602f4a85b9dbb1b4dcd52ad",
 #     urls = ["https://antsys-sparrow-data.cn-shanghai-alipay-office.oss-alipay.aliyuncs.com/sparrow/public/tools/swift/lib_InternalSwiftSyntaxParser/5.5.3-alios7/lib_InternalSwiftSyntaxParser.zip"],
 # )
+
+
+# third_party_typescript 4.9.5-raw Release
+# Replace with the third_party_typescript commit you want to use.
+third_party_typescript_commit = "5bef9c4ec07035be8167d8190f6df47275b81058"
+third_party_typescript_SHA256 = "e703a2ccb546e7fa78f0a0681cbc9569991b807247ff903bf6f0fe5304ec0fe9"
+
+http_archive(
+    name = "ohos_typescript_src",
+    sha256 = third_party_typescript_SHA256,
+    strip_prefix = "third_party_typescript-" + third_party_typescript_commit,
+    build_file = "//:external/rules_ohos_typescript/ohos_typescript.BUILD",
+    urls = [
+        "https://gitee.com/openharmony/third_party_typescript/repository/archive/{commit}.zip".format(commit = third_party_typescript_commit),
+        "https://antsys-sparrow-data.cn-shanghai-alipay-office.oss-alipay.aliyuncs.com/sparrow/public/tools/ohos-typescript/third_party_typescript-{commit}.zip".format(commit = third_party_typescript_commit),
+    ],
+)

external/rules_ohos_typescript/ohos_typescript.BUILD

@@ -0,0 +1,13 @@
+filegroup(
+    name = "ohos_typescript",
+    srcs = glob(["**/*"]),
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "compile_typescript",
+    srcs = [
+        "compile_typescript.py",
+    ],
+    visibility = ["//visibility:public"],
+)

godel-script/README.md

@@ -52,24 +52,39 @@ Structure of this project:
     +-- src             godel-frontend source code
 ```
 
+### Environment
+
 Need C++ standard at least `-std=c++17`.
 
+On Ubuntu, you are expected to install the following packages before compiling.
+
+```bash
+sudo apt install -y git build-essential libffi-dev m4 cmake libsqlite3-dev zlib1g-dev
+```
+
+For convenience, we recommend directly using the [Dev Container plugin](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) of VSCode. The configuration files are in `godel-script/.devcontainer/`.
+
+
 ### Apply Patch On Soufflé Submodule
 
-GödelScript uses a self-modified soufflé from a much older branch of public soufflé,
-now we use patch to make sure it could be built successfully.
+GödelScript uses a self-modified soufflé from a much older branch of public soufflé. Use these commands to clone.
+
+```bash
+git submodule init
+git submodule update --recursive
+```
 
-Use this command to apply patch:
+Now we use patch to make sure it could be built successfully. Use these commands to apply patch:
 
 ```bash
-cd souffle
+cd godel-backend/souffle
 git am ../0001-init-self-used-souffle-from-public-souffle.patch
 ```
 
 Use these commands to revert:
 
 ```bash
-cd souffle
+cd godel-backend/souffle
 git apply -R ../0001-init-self-used-souffle-from-public-souffle.patch
 git reset HEAD~
 ```
@@ -79,8 +94,9 @@ git reset HEAD~
 Use command below:
 
 ```bash
-mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release
-make -j6
+mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE:STRING=Release 
+cmake --build .
 ```
 
 After building, you'll find `build/godel` in the `build` folder.

language/arkts/BUILD

@@ -0,0 +1,11 @@
+load("//:visibility.bzl", "PUBLIC_VISIBILITY")
+
+# Using filegroup is encouraged instead of referencing directories directly. The latter is unsound.
+# When combined with glob, filegroup can ensure that all files are explicitly known to the build system.
+
+# Only one level of depth is matched, and the templates in the resource directory under lib-gen will not be matched.
+filegroup(
+    name = "lib",
+    srcs = glob(["lib/*"]),
+    visibility = ["//visibility:public"],
+)

language/arkts/extractor/.gitignore

@@ -0,0 +1,26 @@
+# IDEs
+.vscode
+.idea
+
+# Dependency directory
+node_modules/
+
+# Build
+dist/
+out/
+
+# Bazel
+bazel-*
+
+# SQLite
+*.db
+*.db-journal
+
+# Coverage reports
+coverage
+
+# API keys and secrets
+.env
+
+# OS metadata
+.DS_Store

language/arkts/extractor/BUILD

@@ -0,0 +1,62 @@
+load("//:visibility.bzl", "PUBLIC_VISIBILITY")
+
+genrule(
+    name = "ohos_typescript_compile",
+    srcs = [],
+    tools = [
+        "@ohos_typescript_src//:compile_typescript"
+    ],
+    outs = ["ohos-typescript"],
+    cmd = """
+    export OHOS_TYPESCRIPT_SRC=$$(dirname $$(pwd)/$(execpath @ohos_typescript_src//:compile_typescript))
+    export OUTPUT_DIR=$$(dirname $$(pwd)/$@)
+    export OHOS_TYPESCRIPT_SRC_CP=$$OUTPUT_DIR/ohos_typescript_src
+    cp -r $$OHOS_TYPESCRIPT_SRC $$OUTPUT_DIR
+    mkdir $(OUTS)
+    # must change dir to src, then execute compile script
+    cd $$OHOS_TYPESCRIPT_SRC_CP
+    python3 compile_typescript.py . $$OUTPUT_DIR/ohos-typescript
+    echo ohos_typescript build success: $$OUTPUT_DIR/ohos-typescript
+    """,
+)
+
+genrule(
+    name = "build-coref-arkts-src-extractor",
+    srcs = glob(["src/**/*.ts"]) + glob(["sdk/**/*"]) + [
+        "schema.prisma",
+        "package.json",
+        "package-lock.json",
+        "paths.json",
+        "tsconfig.json",
+        "@pkg_cache//:node-bin",
+    ] + [":ohos_typescript_compile"],
+    tools = [
+        "@nodejs_host//:npm_bin",
+    ],
+    outs = [
+        "coref-arkts-src-extractor",
+    ],
+    cmd = """
+    NPM_PATH="$$(pwd)/$(execpath @nodejs_host//:npm_bin)"
+    echo NPM_PATH: $$NPM_PATH
+
+    export PKG_CACHE_PATH=$$(dirname $$(dirname $$(pwd)/$(execpath @pkg_cache//:node-bin)))
+    echo PKG_CACHE_PATH: $$PKG_CACHE_PATH
+
+    OUT_PATH="$$(pwd)/$@"
+    echo OUT_PATH: $$OUT_PATH
+
+    # 注意在下面cd前对需要拷贝的typescript进行路径确认
+    export THIRD_TYPESCIRPT_PATH=$$(pwd)/$(location :ohos_typescript_compile)
+    echo THIRD_TYPESCIRPT_PATH: $$THIRD_TYPESCIRPT_PATH
+
+    cd language/arkts/extractor
+    
+    $$NPM_PATH i --registry https://registry.antgroup-inc.cn
+    cp -rf $$THIRD_TYPESCIRPT_PATH ./node_modules/
+    $$NPM_PATH run pkg
+    mv coref-arkts-src-extractor $$OUT_PATH
+    """,
+    executable = 1,
+    visibility = ["//visibility:public"],
+)