From 38c4431127c4819be42b03ab544246ad0ae304d8 Mon Sep 17 00:00:00 2001 From: weixiang1862 Date: Fri, 3 Nov 2023 08:42:32 +0800 Subject: [PATCH] Remove e2e and add another UT to check extract result. --- .github/workflows/skywalking.yaml | 3 - .../graphql/resolver/LogTestQueryTest.java | 30 +++++++++ .../log/extract-timestamp/docker-compose.yml | 64 ------------------- .../cases/log/extract-timestamp/e2e.yaml | 47 -------------- .../cases/log/extract-timestamp/lal.yaml | 36 ----------- 5 files changed, 30 insertions(+), 150 deletions(-) delete mode 100644 test/e2e-v2/cases/log/extract-timestamp/docker-compose.yml delete mode 100644 test/e2e-v2/cases/log/extract-timestamp/e2e.yaml delete mode 100644 test/e2e-v2/cases/log/extract-timestamp/lal.yaml diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index b6261bd5a305..96b3a0736bce 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -440,9 +440,6 @@ jobs: config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml env: ES_VERSION=8.8.1 - - name: Log Extract Timestamp - config: test/e2e-v2/cases/log/extract-timestamp/e2e.yaml - - name: Trace Profiling BanyanDB config: test/e2e-v2/cases/profiling/trace/banyandb/e2e.yaml - name: Trace Profiling H2 diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java b/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java index b60894dee4d5..035e6568e467 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java @@ -135,4 +135,34 @@ public void test() throws Exception { assertEquals(1, response.getMetrics().iterator().next().getValue()); assertEquals(12312313, response.getMetrics().iterator().next().getTimestamp()); } + + @Test + public void testExtractPatternedTimestamp() throws Exception { + when(config.isEnableLogTestTool()).thenReturn(true); + final LogTestQuery query = new LogTestQuery(moduleManager, config); + final LogTestRequest request = new LogTestRequest(); + request.setLog("" + + "{" + + " body: {" + + " json: {" + + " json: '{\"request\": \"GET /l HTTP/1.1\",\"time\": \"2023-11-02T12:39:36+00:00\",\"status\": \"404\",\"request_time\":\"0.000\"}'" + + " }" + + " }," + + " type: JSON," + + " timestamp: 12312313," + + " service: 'test'" + + "}"); + request.setDsl("" + + "filter {\n" + + " json {" + + " }\n" + + " extractor {\n" + + " timestamp parsed.time as String, \"yyyy-MM-dd'T'HH:mm:ssXXX\"\n" + + " }\n" + + " sink {\n" + + " }\n" + + "}"); + final LogTestResponse response = query.test(request); + assertEquals(1698928776000L, response.getLog().getTimestamp()); + } } diff --git a/test/e2e-v2/cases/log/extract-timestamp/docker-compose.yml b/test/e2e-v2/cases/log/extract-timestamp/docker-compose.yml deleted file mode 100644 index fb3912656fc1..000000000000 --- a/test/e2e-v2/cases/log/extract-timestamp/docker-compose.yml +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: '2.1' - -services: - h2db: - build: - context: . - dockerfile: ../../../script/dockerfile/Dockerfile.h2 - networks: - - e2e - expose: - - 1521 - healthcheck: - test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/1521"] - interval: 5s - timeout: 60s - retries: 120 - - oap: - extends: - file: ../log-base-compose.yml - service: oap - environment: - SW_STORAGE: h2 - SW_STORAGE_H2_URL: jdbc:h2:tcp://h2db:1521/skywalking-oap-db;DATABASE_TO_UPPER=FALSE - volumes: - - ./lal.yaml:/skywalking/config/lal/test.yaml - - ../log-mal.yaml:/skywalking/config/log-mal-rules/test.yaml - depends_on: - h2db: - condition: service_healthy - ports: - - 12800 - networks: - - e2e - - provider: - extends: - file: ../../../script/docker-compose/base-compose.yml - service: provider - ports: - - 9090 - networks: - - e2e - depends_on: - oap: - condition: service_healthy - -networks: - e2e: diff --git a/test/e2e-v2/cases/log/extract-timestamp/e2e.yaml b/test/e2e-v2/cases/log/extract-timestamp/e2e.yaml deleted file mode 100644 index ca75d051fd7d..000000000000 --- a/test/e2e-v2/cases/log/extract-timestamp/e2e.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is used to show how to write configuration files and can be used to test. - -setup: - env: compose - file: docker-compose.yml - timeout: 20m - init-system-environment: ../../../script/env - steps: - - name: set PATH - command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH - - name: install yq - command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq - - name: install swctl - command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl - -trigger: - action: http - interval: 3s - times: 10 - url: http://${provider_host}:${provider_9090}/logs/trigger - method: GET - -verify: - # verify with retry strategy - retry: - # max retry count - count: 20 - # the interval between two retries, in millisecond. - interval: 10s - cases: - - includes: - - ../log-cases.yaml diff --git a/test/e2e-v2/cases/log/extract-timestamp/lal.yaml b/test/e2e-v2/cases/log/extract-timestamp/lal.yaml deleted file mode 100644 index bd5ae325bf38..000000000000 --- a/test/e2e-v2/cases/log/extract-timestamp/lal.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -rules: - - name: example - layer: GENERAL - dsl: | - filter { - text { - abortOnFailure false // for test purpose, we want to persist all logs - regexp $/(?s)(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}) \[TID:(?.+?)] \[(?.+?)] (?\w{4,}) (?.{1,36}) (?.+)/$ - } - extractor { - timestamp parsed.timestamp as String, "yyyy-MM-dd HH:mm:ss.SSS" - metrics { - timestamp log.timestamp as Long - labels level: parsed.level, service: log.service, instance: log.serviceInstance - name "log_count" - value 1 - } - } - sink { - } - }