From 588f8858011b8c59fb7b88b9e342ab33a2232cb0 Mon Sep 17 00:00:00 2001 From: rahul-privado Date: Tue, 4 Jul 2023 22:05:51 +0530 Subject: [PATCH 1/7] Parser fail prints --- .../rubysrc2cpg/astcreation/AstCreator.scala | 18 +++++++++--------- .../astcreation/AstForStatementsCreator.scala | 8 ++++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index ce3ca72278ce..b990d918ff9e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -241,7 +241,7 @@ class AstCreator( val node = createIdentifierWithScope(ctx, varSymbol.getText, varSymbol.getText, Defines.Any, List(Defines.Any)) Seq(Ast(node)) case _ => - logger.error("astForSingleLeftHandSideContext() All contexts mismatched.") + logger.error(s"astForSingleLeftHandSideContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } @@ -366,7 +366,7 @@ class AstCreator( case ctx: ChainedInvocationWithoutArgumentsPrimaryContext => astForChainedInvocationWithoutArgumentsPrimaryContext(ctx) case _ => - logger.error("astForPrimaryContext() All contexts mismatched.") + logger.error(s"astForPrimaryContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } @@ -390,7 +390,7 @@ class AstCreator( case ctx: MultipleAssignmentExpressionContext => astForMultipleAssignmentExpressionContext(ctx) case ctx: IsDefinedExpressionContext => Seq(astForIsDefinedExpression(ctx)) case _ => - logger.error("astForExpressionContext() All contexts mismatched.") + logger.error(s"astForExpressionContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } @@ -441,7 +441,7 @@ class AstCreator( case ctx: RubyParser.SplattingOnlyIndexingArgumentsContext => astForSplattingArgumentContext(ctx.splattingArgument()) case _ => - logger.error("astForIndexingArgumentsContext() All contexts mismatched.") + logger.error(s"astForIndexingArgumentsContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } @@ -653,7 +653,7 @@ class AstCreator( case ctx: GroupedLeftHandSideOnlyMultipleLeftHandSideContext => astForGroupedLeftHandSideContext(ctx.groupedLeftHandSide()) case _ => - logger.error("astForMultipleLeftHandSideContext() All contexts mismatched.") + logger.error(s"astForMultipleLeftHandSideContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } @@ -761,7 +761,7 @@ class AstCreator( .withChildren(astForArguments(ctx.arguments())) ) case _ => - logger.error("astForInvocationWithoutParenthesesContext() All contexts mismatched.") + logger.error(s"astForInvocationWithoutParenthesesContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } @@ -1011,7 +1011,7 @@ class AstCreator( case ctx: SimpleMethodNamePartContext => astForSimpleMethodNamePartContext(ctx) case ctx: SingletonMethodNamePartContext => astForSingletonMethodNamePartContext(ctx) case _ => - logger.error("astForMethodNamePartContext() All contexts mismatched.") + logger.error(s"astForMethodNamePartContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } @@ -1358,7 +1358,7 @@ class AstCreator( val primaryAsts = astForPrimaryContext(ctx.primary()) primaryAsts ++ methodNameAsts ++ argsAsts ++ doBlockAsts case _ => - logger.error("astForCommandWithDoBlockContext() All contexts mismatched.") + logger.error(s"astForCommandWithDoBlockContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } @@ -1392,7 +1392,7 @@ class AstCreator( case ctx: ChainedCommandWithDoBlockOnlyArgumentsWithParenthesesContext => astForChainedCommandWithDoBlockContext(ctx.chainedCommandWithDoBlock()) case _ => - logger.error("astForArgumentsWithParenthesesContext() All contexts mismatched.") + logger.error(s"astForArgumentsWithParenthesesContext() $filename, ${ctx.getText} All contexts mismatched.") Seq(Ast()) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index 9f6b0afa29a0..077c8ae463c2 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -1,17 +1,19 @@ package io.joern.rubysrc2cpg.astcreation import better.files.File -import io.joern.rubysrc2cpg.parser.RubyParser._ +import io.joern.rubysrc2cpg.parser.RubyParser.* import io.joern.rubysrc2cpg.passes.Defines import io.joern.x2cpg.Ast import io.joern.x2cpg.Defines.DynamicCallUnknownFullName import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.{NewBlock, NewCall, NewControlStructure, NewImport, NewLiteral} +import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.CollectionHasAsScala trait AstForStatementsCreator { this: AstCreator => + private val logger = LoggerFactory.getLogger(this.getClass) protected def astForAliasStatement(ctx: AliasStatementContext): Ast = { val aliasName = ctx.definedMethodNameOrSymbol(0).getText.substring(1) val methodName = ctx.definedMethodNameOrSymbol(1).getText.substring(1) @@ -114,7 +116,9 @@ trait AstForStatementsCreator { this: AstCreator => case ctx: NotExpressionOrCommandContext => Seq(astForNotKeywordExpressionOrCommand(ctx)) case ctx: OrAndExpressionOrCommandContext => Seq(astForOrAndExpressionOrCommand(ctx)) case ctx: ExpressionExpressionOrCommandContext => astForExpressionContext(ctx.expression()) - case _ => Seq(Ast()) + case _ => + logger.error(s"astForExpressionOrCommand() $filename, ${ctx.getText} All contexts mismatched.") + Seq(Ast()) } protected def astForNotKeywordExpressionOrCommand(ctx: NotExpressionOrCommandContext): Ast = { From 8c096dc12c4dfc939c1b5a40f7efbf75c8623eaf Mon Sep 17 00:00:00 2001 From: rahul-privado Date: Wed, 5 Jul 2023 16:18:20 +0530 Subject: [PATCH 2/7] More tests --- .../rubysrc2cpg/dataflow/DataFlowTests.scala | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala index bc2d41c65424..c2592d49099e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala @@ -1010,6 +1010,45 @@ class DataFlowTests extends DataFlowCodeToCpgSuite { } } + "Data flow for begin/rescue with data flow through the exception" should { + val cpg = code(""" + |x = "Exception message: " + |begin + |1/0 + |rescue ZeroDivisionError => e + | y = x + e.message + | puts y + |end + | + |""".stripMargin) + + "find flows to the sink" in { + val source = cpg.identifier.name("x").l + val sink = cpg.call.name("puts").l + sink.reachableByFlows(source).size shouldBe 2 + } + } + + "Data flow for begin/rescue with data flow through block with multiple exceptions being caught" should { + val cpg = code(""" + |x = 1 + |y = 10 + |begin + |1/0 + |rescue SystemCallError, ZeroDivisionError + | y = x + 100 + |end + | + |puts y + |""".stripMargin) + + "find flows to the sink" in { + val source = cpg.identifier.name("x").l + val sink = cpg.call.name("puts").l + sink.reachableByFlows(source).size shouldBe 2 + } + } + "Data flow for begin/rescue with sink in function without begin" ignore { val cpg = code(""" |def foo(arg) From 1dcbc614029b7a8d5339000b0c5637a296d98693 Mon Sep 17 00:00:00 2001 From: rahul-privado Date: Wed, 5 Jul 2023 16:24:07 +0530 Subject: [PATCH 3/7] More tests --- .../rubysrc2cpg/dataflow/DataFlowTests.scala | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala index c2592d49099e..8128c82a068c 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala @@ -1010,6 +1010,30 @@ class DataFlowTests extends DataFlowCodeToCpgSuite { } } + "Data flow for begin/rescue with sink in ensure" should { + val cpg = code(""" + |x = 1 + |begin + | puts "in begin" + |rescue SomeException + | puts "SomeException occurred" + |rescue => exceptionVar + | puts "Caught exception in variable #{exceptionVar}" + |rescue + | puts "In rescue all" + |ensure + | puts x + |end + | + |""".stripMargin) + + "find flows to the sink" in { + val source = cpg.identifier.name("x").l + val sink = cpg.call.name("puts").l + sink.reachableByFlows(source).size shouldBe 2 + } + } + "Data flow for begin/rescue with data flow through the exception" should { val cpg = code(""" |x = "Exception message: " From 13bd81a1b902c89a4b770c42b6059b70e88b8593 Mon Sep 17 00:00:00 2001 From: rahul-privado Date: Wed, 5 Jul 2023 17:20:37 +0530 Subject: [PATCH 4/7] Optional packing in block for compound statement --- .../io/joern/rubysrc2cpg/astcreation/AstCreator.scala | 2 +- .../rubysrc2cpg/astcreation/AstForStatementsCreator.scala | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index cc2df9921b11..a5e22b12aaf7 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -1050,7 +1050,7 @@ class AstCreator( } def astForBodyStatementContext(ctx: BodyStatementContext, addReturnNode: Boolean = false): Seq[Ast] = { - val compoundStatementAsts = astForCompoundStatement(ctx.compoundStatement()) + val compoundStatementAsts = astForCompoundStatement(ctx.compoundStatement(), false) val compoundStatementAstsWithReturn = if (addReturnNode && compoundStatementAsts.size > 0) { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index 4e0803b7bf23..2bbfdc4c0728 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -80,9 +80,13 @@ trait AstForStatementsCreator { controlStructureAst(throwNode, rhs.headOption, lhs) } - protected def astForCompoundStatement(ctx: CompoundStatementContext): Seq[Ast] = { + protected def astForCompoundStatement(ctx: CompoundStatementContext, packInBlock: Boolean = true): Seq[Ast] = { val stmtAsts = Option(ctx.statements()).map(astForStatements).getOrElse(Seq()) - Seq(blockAst(blockNode(ctx), stmtAsts.toList)) + if (packInBlock) { + Seq(blockAst(blockNode(ctx), stmtAsts.toList)) + } else { + stmtAsts + } } protected def astForStatements(ctx: StatementsContext): Seq[Ast] = { From 2fcbd2b88e631ec361df3e1b2b2288be19cdd246 Mon Sep 17 00:00:00 2001 From: rahul-privado Date: Wed, 5 Jul 2023 17:29:11 +0530 Subject: [PATCH 5/7] Test for else --- .../rubysrc2cpg/dataflow/DataFlowTests.scala | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala index 8128c82a068c..e108b99be12c 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala @@ -944,6 +944,29 @@ class DataFlowTests extends DataFlowCodeToCpgSuite { } } + "Data flow for begin/rescue with sink in else" should { + val cpg = code(""" + |x = 1 + |begin + | puts "In begin" + |rescue SomeException + | puts "SomeException occurred" + |rescue => exceptionVar + | puts "Caught exception in variable #{exceptionVar}" + |rescue + | puts "Catch-all block" + |else + | puts x + |end + |""".stripMargin) + + "find flows to the sink" in { + val source = cpg.identifier.name("x").l + val sink = cpg.call.name("puts").l + sink.reachableByFlows(source).size shouldBe 2 + } + } + "Data flow for begin/rescue with sink in rescue" should { val cpg = code(""" |x = 1 From 2ccf9fe2eaaa6b3aec0075d01fccb849bddbc2f6 Mon Sep 17 00:00:00 2001 From: rahul-privado Date: Wed, 5 Jul 2023 17:33:04 +0530 Subject: [PATCH 6/7] Flag passing for return node --- .../scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index a5e22b12aaf7..7971fdd80485 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -1050,7 +1050,7 @@ class AstCreator( } def astForBodyStatementContext(ctx: BodyStatementContext, addReturnNode: Boolean = false): Seq[Ast] = { - val compoundStatementAsts = astForCompoundStatement(ctx.compoundStatement(), false) + val compoundStatementAsts = astForCompoundStatement(ctx.compoundStatement(), !addReturnNode) val compoundStatementAstsWithReturn = if (addReturnNode && compoundStatementAsts.size > 0) { From ca441b00ec4b70ef01011518e0813381d8a56a9b Mon Sep 17 00:00:00 2001 From: rahul-privado Date: Wed, 5 Jul 2023 17:36:39 +0530 Subject: [PATCH 7/7] Disabled test case failing parsing --- .../scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala index e108b99be12c..f5b064293a9b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala @@ -1057,7 +1057,8 @@ class DataFlowTests extends DataFlowCodeToCpgSuite { } } - "Data flow for begin/rescue with data flow through the exception" should { + // parsing issue. comment out when fixed + "Data flow for begin/rescue with data flow through the exception" ignore { val cpg = code(""" |x = "Exception message: " |begin