td_ddlなどのDigdagのオペレータにおいて、複数のテーブルに対してまとめて操作するには、['xxxx','yyyy']どいった配列にしなければならない。 その際に、各テーブルに日付などの要素を入れたい場合には下記のように行う。 つまり気合いで正規表現。
_export:
tbls:
- main_tmp_table
- main_tmp2_table
- main_tmp3_table
- main_tmp4_table
ymd: 20200128
td:
database: support
+step1:
td_ddl>:
create_tables: ${JSON.stringify(tbls).replace(/(\w+)/g, '$1_'+ymd)}
$ digdag run main.dig --rerun 2020-02-06 21:04:00 +0900: Digdag v0.9.41 2020-02-06 21:04:03 +0900 [WARN] (main): Reusing the last session time 2020-02-06T00:00:00+00:00. 2020-02-06 21:04:03 +0900 [INFO] (main): Using session /Users/toru/Desktop/map_dig/.digdag/status/20200206T000000+0000. 2020-02-06 21:04:03 +0900 [INFO] (main): Starting a new session project id=1 workflow name=main session_time=2020-02-06T00:00:00+00:00 2020-02-06 21:04:04 +0900 [INFO] (0017@[0:default]+main+step1): td_ddl>: 2020-02-06 21:04:04 +0900 [INFO] (0017@[0:default]+main+step1): td-client version: 0.9.0 2020-02-06 21:04:04 +0900 [INFO] (0017@[0:default]+main+step1): Creating TD table support.main_tmp_table_20200128 2020-02-06 21:04:06 +0900 [INFO] (0017@[0:default]+main+step1): Creating TD table support.main_tmp2_table_20200128 2020-02-06 21:04:07 +0900 [INFO] (0017@[0:default]+main+step1): Creating TD table support.main_tmp3_table_20200128 2020-02-06 21:04:07 +0900 [INFO] (0017@[0:default]+main+step1): Creating TD table support.main_tmp4_table_20200128 Success. Task state is saved at /Users/toru/Desktop/map_dig/.digdag/status/20200206T000000+0000 directory. * Use --session <daily | hourly | "yyyy-MM-dd[ HH:mm:ss]"> to not reuse the last session time. * Use --rerun, --start +NAME, or --goal +NAME argument to rerun skipped tasks.
Map関数などを使うと、下記のエラーが発生する。 nashornの不具合っぽいので、みなかったことにする。
2020-02-06 19:51:06 +0900 [ERROR] (0017@[0:default]+main+chk): Task failed with unexpected error: Unexpected error happened in ConfigEvalEngine: Failed generating bytecode for <function>:5
java.lang.RuntimeException: Unexpected error happened in ConfigEvalEngine: Failed generating bytecode for <function>:5
at io.digdag.core.agent.OperatorManager.runWithWorkspace(OperatorManager.java:207)
at io.digdag.core.agent.OperatorManager.lambda$runWithHeartbeat$2(OperatorManager.java:137)
at io.digdag.core.agent.LocalWorkspaceManager.withExtractedArchive(LocalWorkspaceManager.java:25)
at io.digdag.core.agent.OperatorManager.runWithHeartbeat(OperatorManager.java:135)
at io.digdag.core.agent.OperatorManager.run(OperatorManager.java:119)
at io.digdag.cli.Run$OperatorManagerWithSkip.run(Run.java:687)
at io.digdag.core.agent.MultiThreadAgent.lambda$null$0(MultiThreadAgent.java:127)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: Failed generating bytecode for <function>:5
at jdk.nashorn.internal.codegen.CompilationPhase$BytecodeGenerationPhase.transform(CompilationPhase.java:431)
at jdk.nashorn.internal.codegen.CompilationPhase.apply(CompilationPhase.java:624)
at jdk.nashorn.internal.codegen.Compiler.compile(Compiler.java:655)
at jdk.nashorn.internal.runtime.RecompilableScriptFunctionData.compileTypeSpecialization(RecompilableScriptFunctionData.java:725)
at jdk.nashorn.internal.runtime.RecompilableScriptFunctionData.getBest(RecompilableScriptFunctionData.java:905)
at jdk.nashorn.internal.runtime.ScriptFunctionData.getGeneric(ScriptFunctionData.java:384)
at jdk.nashorn.internal.runtime.ScriptFunctionData.createGenericInvoker(ScriptFunctionData.java:282)
at jdk.nashorn.internal.runtime.ScriptFunctionData.getGenericInvoker(ScriptFunctionData.java:276)
at jdk.nashorn.internal.runtime.ScriptFunctionData.invoke(ScriptFunctionData.java:620)
at jdk.nashorn.internal.runtime.ScriptFunction.invoke(ScriptFunction.java:494)
at jdk.nashorn.internal.runtime.ScriptRuntime.apply(ScriptRuntime.java:393)
at jdk.nashorn.internal.objects.NativeFunction.call(NativeFunction.java:192)
at jdk.nashorn.internal.scripts.Script$Recompilation$36$62AA$\^eval\_.template(<eval>:53)
at jdk.nashorn.internal.runtime.ScriptFunctionData.invoke(ScriptFunctionData.java:641)
at jdk.nashorn.internal.runtime.ScriptFunction.invoke(ScriptFunction.java:494)
at jdk.nashorn.internal.runtime.ScriptRuntime.apply(ScriptRuntime.java:393)
at jdk.nashorn.api.scripting.ScriptObjectMirror.callMember(ScriptObjectMirror.java:199)
at jdk.nashorn.api.scripting.NashornScriptEngine.invokeImpl(NashornScriptEngine.java:383)
at jdk.nashorn.api.scripting.NashornScriptEngine.invokeFunction(NashornScriptEngine.java:190)
at io.digdag.core.agent.ConfigEvalEngine.invokeTemplate(ConfigEvalEngine.java:112)
at io.digdag.core.agent.ConfigEvalEngine.access$200(ConfigEvalEngine.java:30)
at io.digdag.core.agent.ConfigEvalEngine$Context.evalValue(ConfigEvalEngine.java:206)
at io.digdag.core.agent.ConfigEvalEngine$Context.evalObjectRecursive(ConfigEvalEngine.java:164)
at io.digdag.core.agent.ConfigEvalEngine$Context.access$000(ConfigEvalEngine.java:132)
at io.digdag.core.agent.ConfigEvalEngine.eval(ConfigEvalEngine.java:78)
at io.digdag.core.agent.OperatorManager.runWithWorkspace(OperatorManager.java:195)
... 11 common frames omitted
Caused by: java.lang.AssertionError: jdk.nashorn.internal.ir.BinaryNode
at jdk.nashorn.internal.codegen.CodeGenerator$2.enterDefault(CodeGenerator.java:1226)
at jdk.nashorn.internal.ir.visitor.NodeOperatorVisitor.enterBIND(NodeOperatorVisitor.java:744)
at jdk.nashorn.internal.ir.visitor.NodeOperatorVisitor.enterBinaryNode(NodeOperatorVisitor.java:137)
at jdk.nashorn.internal.ir.BinaryNode.accept(BinaryNode.java:335)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:857)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:833)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpressionUnbounded(CodeGenerator.java:568)
at jdk.nashorn.internal.codegen.CodeGenerator.loadArgs(CodeGenerator.java:1421)
at jdk.nashorn.internal.codegen.CodeGenerator.access$5100(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$3$4.loadStack(CodeGenerator.java:1605)
at jdk.nashorn.internal.codegen.CodeGenerator$OptimisticOperation.emit(CodeGenerator.java:4642)
at jdk.nashorn.internal.codegen.CodeGenerator$OptimisticOperation.emit(CodeGenerator.java:4627)
at jdk.nashorn.internal.codegen.CodeGenerator$3.enterAccessNode(CodeGenerator.java:1611)
at jdk.nashorn.internal.ir.AccessNode.accept(AccessNode.java:66)
at jdk.nashorn.internal.codegen.CodeGenerator.loadCallNode(CodeGenerator.java:1434)
at jdk.nashorn.internal.codegen.CodeGenerator.access$2100(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$2.enterCallNode(CodeGenerator.java:1002)
at jdk.nashorn.internal.ir.CallNode.accept(CallNode.java:177)
at jdk.nashorn.internal.ir.LexicalContextNode$Acceptor.accept(LexicalContextNode.java:57)
at jdk.nashorn.internal.ir.LexicalContextExpression.accept(LexicalContextExpression.java:47)
at jdk.nashorn.internal.ir.CallNode.accept(CallNode.java:41)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:857)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:833)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpressionAsType(CodeGenerator.java:829)
at jdk.nashorn.internal.codegen.CodeGenerator$16.evaluate(CodeGenerator.java:3870)
at jdk.nashorn.internal.codegen.CodeGenerator$Store.store(CodeGenerator.java:4524)
at jdk.nashorn.internal.codegen.CodeGenerator.loadASSIGN(CodeGenerator.java:3872)
at jdk.nashorn.internal.codegen.CodeGenerator.access$900(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$2.enterASSIGN(CodeGenerator.java:919)
at jdk.nashorn.internal.ir.visitor.NodeOperatorVisitor.enterBinaryNode(NodeOperatorVisitor.java:113)
at jdk.nashorn.internal.ir.BinaryNode.accept(BinaryNode.java:335)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:857)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:833)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpressionUnbounded(CodeGenerator.java:568)
at jdk.nashorn.internal.codegen.CodeGenerator.nullCheck(CodeGenerator.java:2726)
at jdk.nashorn.internal.codegen.CodeGenerator.loadRuntimeNode(CodeGenerator.java:2809)
at jdk.nashorn.internal.codegen.CodeGenerator.access$4300(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$2.enterRuntimeNode(CodeGenerator.java:1193)
at jdk.nashorn.internal.ir.RuntimeNode.accept(RuntimeNode.java:436)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:857)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:833)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpressionAsBoolean(CodeGenerator.java:576)
at jdk.nashorn.internal.codegen.BranchOptimizer.loadTestAndJump(BranchOptimizer.java:166)
at jdk.nashorn.internal.codegen.BranchOptimizer.branchOptimizer(BranchOptimizer.java:162)
at jdk.nashorn.internal.codegen.BranchOptimizer.execute(BranchOptimizer.java:57)
at jdk.nashorn.internal.codegen.CodeGenerator.emitBranch(CodeGenerator.java:2206)
at jdk.nashorn.internal.codegen.CodeGenerator.loadTernaryNode(CodeGenerator.java:4251)
at jdk.nashorn.internal.codegen.CodeGenerator.access$2300(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$2.enterTernaryNode(CodeGenerator.java:1013)
at jdk.nashorn.internal.ir.TernaryNode.accept(TernaryNode.java:70)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:857)
at jdk.nashorn.internal.codegen.CodeGenerator.loadBinaryOperands(CodeGenerator.java:618)
at jdk.nashorn.internal.codegen.CodeGenerator.access$7200(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$15.loadStack(CodeGenerator.java:3772)
at jdk.nashorn.internal.codegen.CodeGenerator$OptimisticOperation.emit(CodeGenerator.java:4642)
at jdk.nashorn.internal.codegen.CodeGenerator$OptimisticOperation.emit(CodeGenerator.java:4627)
at jdk.nashorn.internal.codegen.CodeGenerator.loadADD(CodeGenerator.java:3779)
at jdk.nashorn.internal.codegen.CodeGenerator$2.enterADD(CodeGenerator.java:1019)
at jdk.nashorn.internal.ir.visitor.NodeOperatorVisitor.enterBinaryNode(NodeOperatorVisitor.java:109)
at jdk.nashorn.internal.ir.BinaryNode.accept(BinaryNode.java:335)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:857)
at jdk.nashorn.internal.codegen.CodeGenerator.loadBinaryOperands(CodeGenerator.java:616)
at jdk.nashorn.internal.codegen.CodeGenerator.access$7200(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$15.loadStack(CodeGenerator.java:3772)
at jdk.nashorn.internal.codegen.CodeGenerator$OptimisticOperation.emit(CodeGenerator.java:4642)
at jdk.nashorn.internal.codegen.CodeGenerator$OptimisticOperation.emit(CodeGenerator.java:4627)
at jdk.nashorn.internal.codegen.CodeGenerator.loadADD(CodeGenerator.java:3779)
at jdk.nashorn.internal.codegen.CodeGenerator$2.enterADD(CodeGenerator.java:1019)
at jdk.nashorn.internal.ir.visitor.NodeOperatorVisitor.enterBinaryNode(NodeOperatorVisitor.java:109)
at jdk.nashorn.internal.ir.BinaryNode.accept(BinaryNode.java:335)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:857)
at jdk.nashorn.internal.codegen.CodeGenerator.loadBinaryOperands(CodeGenerator.java:618)
at jdk.nashorn.internal.codegen.CodeGenerator.access$7200(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$BinaryOptimisticSelfAssignment$1.loadStack(CodeGenerator.java:3907)
at jdk.nashorn.internal.codegen.CodeGenerator$OptimisticOperation.emit(CodeGenerator.java:4642)
at jdk.nashorn.internal.codegen.CodeGenerator$BinaryOptimisticSelfAssignment.evaluate(CodeGenerator.java:3913)
at jdk.nashorn.internal.codegen.CodeGenerator$Store.store(CodeGenerator.java:4524)
at jdk.nashorn.internal.codegen.CodeGenerator.loadASSIGN_ADD(CodeGenerator.java:3942)
at jdk.nashorn.internal.codegen.CodeGenerator.access$1000(CodeGenerator.java:178)
at jdk.nashorn.internal.codegen.CodeGenerator$2.enterASSIGN_ADD(CodeGenerator.java:926)
at jdk.nashorn.internal.ir.visitor.NodeOperatorVisitor.enterBinaryNode(NodeOperatorVisitor.java:115)
at jdk.nashorn.internal.ir.BinaryNode.accept(BinaryNode.java:335)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:857)
at jdk.nashorn.internal.codegen.CodeGenerator.loadExpression(CodeGenerator.java:833)
at jdk.nashorn.internal.codegen.CodeGenerator.loadAndDiscard(CodeGenerator.java:3663)
at jdk.nashorn.internal.codegen.CodeGenerator.enterExpressionStatement(CodeGenerator.java:1720)
at jdk.nashorn.internal.ir.ExpressionStatement.accept(ExpressionStatement.java:63)
at jdk.nashorn.internal.ir.Node.accept(Node.java:271)
at jdk.nashorn.internal.ir.Block.accept(Block.java:185)
at jdk.nashorn.internal.ir.LexicalContextNode$Acceptor.accept(LexicalContextNode.java:57)
at jdk.nashorn.internal.ir.Block.accept(Block.java:439)
at jdk.nashorn.internal.codegen.CodeGenerator.enterWithNode(CodeGenerator.java:3534)
at jdk.nashorn.internal.ir.WithNode.accept(WithNode.java:70)
at jdk.nashorn.internal.ir.LexicalContextNode$Acceptor.accept(LexicalContextNode.java:57)
at jdk.nashorn.internal.ir.LexicalContextStatement.accept(LexicalContextStatement.java:55)
at jdk.nashorn.internal.ir.WithNode.accept(WithNode.java:34)
at jdk.nashorn.internal.ir.Node.accept(Node.java:271)
at jdk.nashorn.internal.ir.Block.accept(Block.java:185)
at jdk.nashorn.internal.ir.LexicalContextNode$Acceptor.accept(LexicalContextNode.java:57)
at jdk.nashorn.internal.ir.Block.accept(Block.java:439)
at jdk.nashorn.internal.ir.FunctionNode.accept(FunctionNode.java:336)
at jdk.nashorn.internal.ir.LexicalContextNode$Acceptor.accept(LexicalContextNode.java:57)
at jdk.nashorn.internal.ir.LexicalContextExpression.accept(LexicalContextExpression.java:47)
at jdk.nashorn.internal.ir.FunctionNode.accept(FunctionNode.java:55)
at jdk.nashorn.internal.codegen.CompilationPhase.transformFunction(CompilationPhase.java:628)
at jdk.nashorn.internal.codegen.CompilationPhase.access$100(CompilationPhase.java:55)
at jdk.nashorn.internal.codegen.CompilationPhase$BytecodeGenerationPhase.transform(CompilationPhase.java:418)
... 36 common frames omitted
