datax3 官方案例出错解决
- 测试官方案例报错
[root@slave1 datax]# python bin/datax.py job/._job.json
DataX (DATAX-OPENSOURCE-3.0), From Alibaba !
Copyright (C) 2010-2017, Alibaba Group. All Rights Reserved.
2022-03-23 00:47:05.117 [main] ERROR Engine -
经DataX智能分析,该任务最可能的错误原因是:
com.alibaba.datax.common.exception.DataXException: Code:[Common-00], Describe:[您提供的配置文件存在错误信息，请检查您的作业配置 .] - 配置信息错误. 您提供的配置信息不是合法的JSON格式: syntax error, pos 1 . 请按照标准json格式提供配置信息.
at com.alibaba.datax.common.exception.DataXException.asDataXException(DataXException.java:26)
at com.alibaba.datax.common.util.Configuration.<init>(Configuration.java:1066)
at com.alibaba.datax.common.util.Configuration.from(Configuration.java:79)
at com.alibaba.datax.core.util.ConfigParser.parseJobConfig(ConfigParser.java:75)
at com.alibaba.datax.core.util.ConfigParser.parse(ConfigParser.java:26)
at com.alibaba.datax.core.Engine.entry(Engine.java:137)
at com.alibaba.datax.core.Engine.main(Engine.java:204)
解决：
# 删除plugins的writer和reader目录下的所有._开头文件
[root@slave1 reader]# ll -al
总用量 76
drwxr-xr-x. 21 502 games 4096 3月 23 00:46 .
drwxr-xr-x. 4 502 games 66 3月 23 00:46 ..
drwxr-xr-x. 3 502 games 224 3月 23 00:46 cassandrareader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._cassandrareader
drwxr-xr-x. 3 502 games 214 3月 23 00:46 drdsreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._drdsreader
drwxr-xr-x. 3 502 games 212 3月 23 00:46 ftpreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._ftpreader
drwxr-xr-x. 3 502 games 224 10月 12 2019 hbase094xreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._hbase094xreader
drwxr-xr-x. 3 502 games 222 3月 23 00:46 hbase11xreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._hbase11xreader
drwxr-xr-x. 3 502 games 214 3月 23 00:46 hdfsreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._hdfsreader
drwxr-xr-x. 3 502 games 220 3月 23 00:46 mongodbreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._mongodbreader
drwxr-xr-x. 3 502 games 216 3月 23 00:46 mysqlreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._mysqlreader
drwxr-xr-x. 3 502 games 114 3月 23 00:46 oceanbasev10reader
drwxr-xr-x. 3 502 games 214 3月 23 00:46 odpsreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._odpsreader
drwxr-xr-x. 3 502 games 218 3月 23 00:46 oraclereader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._oraclereader
drwxr-xr-x. 3 502 games 212 3月 23 00:46 ossreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._ossreader
drwxr-xr-x. 3 502 games 212 3月 23 00:46 otsreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._otsreader
drwxr-xr-x. 3 502 games 158 3月 23 00:46 otsstreamreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._otsstreamreader
drwxr-xr-x. 3 502 games 226 3月 23 00:46 postgresqlreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._postgresqlreader
drwxr-xr-x. 3 502 games 216 3月 23 00:46 rdbmsreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._rdbmsreader
drwxr-xr-x. 3 502 games 224 3月 23 00:46 sqlserverreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._sqlserverreader
drwxr-xr-x. 3 502 games 218 3月 23 00:46 streamreader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._streamreader
drwxr-xr-x. 3 502 games 220 3月 23 00:46 txtfilereader
-rwxr-xr-x. 1 502 games 212 10月 12 2019 ._txtfilereader
[root@slave1 reader]# rm -rf ./._*
[root@slave1 reader]# rm -rf ../writer/._*
[root@slave1 reader]# ll -al
总用量 4
drwxr-xr-x. 21 502 games 4096 3月 23 00:52 .
drwxr-xr-x. 4 502 games 66 3月 23 00:46 ..
drwxr-xr-x. 3 502 games 224 3月 23 00:46 cassandrareader
drwxr-xr-x. 3 502 games 214 3月 23 00:46 drdsreader
drwxr-xr-x. 3 502 games 212 3月 23 00:46 ftpreader
drwxr-xr-x. 3 502 games 224 10月 12 2019 hbase094xreader
drwxr-xr-x. 3 502 games 222 3月 23 00:46 hbase11xreader
drwxr-xr-x. 3 502 games 214 3月 23 00:46 hdfsreader
drwxr-xr-x. 3 502 games 220 3月 23 00:46 mongodbreader
drwxr-xr-x. 3 502 games 216 3月 23 00:46 mysqlreader
drwxr-xr-x. 3 502 games 114 3月 23 00:46 oceanbasev10reader
drwxr-xr-x. 3 502 games 214 3月 23 00:46 odpsreader
drwxr-xr-x. 3 502 games 218 3月 23 00:46 oraclereader
drwxr-xr-x. 3 502 games 212 3月 23 00:46 ossreader
drwxr-xr-x. 3 502 games 212 3月 23 00:46 otsreader
drwxr-xr-x. 3 502 games 158 3月 23 00:46 otsstreamreader
drwxr-xr-x. 3 502 games 226 3月 23 00:46 postgresqlreader
drwxr-xr-x. 3 502 games 216 3月 23 00:46 rdbmsreader
drwxr-xr-x. 3 502 games 224 3月 23 00:46 sqlserverreader
drwxr-xr-x. 3 502 games 218 3月 23 00:46 streamreader
drwxr-xr-x. 3 502 games 220 3月 23 00:46 txtfilereader
再次执行官方案例：
[root@slave1 datax]# python bin/datax.py job/job.json
DataX (DATAX-OPENSOURCE-3.0), From Alibaba !
Copyright (C) 2010-2017, Alibaba Group. All Rights Reserved.
2022-03-23 00:54:03.446 [main] INFO VMInfo - VMInfo# operatingSystem class => sun.management.OperatingSystemImpl
2022-03-23 00:54:03.454 [main] INFO Engine - the machine info =>
osInfo: Oracle Corporation 1.8 25.171-b11
jvmInfo: Linux amd64 3.10.0-1127.el7.x86_64
cpu num: 2
totalPhysicalMemory: -0.00G
freePhysicalMemory: -0.00G
maxFileDescriptorCount: -1
currentOpenFileDescriptorCount: -1
GC Names [PS MarkSweep, PS Scavenge]
MEMORY_NAME | allocation_size | init_size
PS Eden Space | 256.00MB | 256.00MB
Code Cache | 240.00MB | 2.44MB
Compressed Class Space | 1,024.00MB | 0.00MB
PS Survivor Space | 42.50MB | 42.50MB
PS Old Gen | 683.00MB | 683.00MB
Metaspace | -0.00MB | 0.00MB
2022-03-23 00:54:03.469 [main] INFO Engine -
{
"content":[
{
"reader":{
"name":"streamreader",
"parameter":{
"column":[
{
"type":"string",
"value":"DataX"
},
{
"type":"long",
"value":19890604
},
{
"type":"date",
"value":"1989-06-04 00:00:00"
},
{
"type":"bool",
"value":true
},
{
"type":"bytes",
"value":"test"
}
],
"sliceRecordCount":100000
}
},
"writer":{
"name":"streamwriter",
"parameter":{
"encoding":"UTF-8",
"print":false
}
}
}
],
"setting":{
"errorLimit":{
"percentage":0.02,
"record":0
},
"speed":{
"byte":10485760
}
}
}
2022-03-23 00:54:03.487 [main] WARN Engine - prioriy set to 0, because NumberFormatException, the value is: null
2022-03-23 00:54:03.489 [main] INFO PerfTrace - PerfTrace traceId=job_-1, isEnable=false, priority=0
2022-03-23 00:54:03.489 [main] INFO JobContainer - DataX jobContainer starts job.
2022-03-23 00:54:03.492 [main] INFO JobContainer - Set jobId = 0
2022-03-23 00:54:03.513 [job-0] INFO JobContainer - jobContainer starts to do prepare ...
2022-03-23 00:54:03.513 [job-0] INFO JobContainer - DataX Reader.Job [streamreader] do prepare work .
2022-03-23 00:54:03.514 [job-0] INFO JobContainer - DataX Writer.Job [streamwriter] do prepare work .
2022-03-23 00:54:03.514 [job-0] INFO JobContainer - jobContainer starts to do split ...
2022-03-23 00:54:03.515 [job-0] INFO JobContainer - Job set Max-Byte-Speed to 10485760 bytes.
2022-03-23 00:54:03.516 [job-0] INFO JobContainer - DataX Reader.Job [streamreader] splits to [1] tasks.
2022-03-23 00:54:03.516 [job-0] INFO JobContainer - DataX Writer.Job [streamwriter] splits to [1] tasks.
2022-03-23 00:54:03.535 [job-0] INFO JobContainer - jobContainer starts to do schedule ...
2022-03-23 00:54:03.541 [job-0] INFO JobContainer - Scheduler starts [1] taskGroups.
2022-03-23 00:54:03.543 [job-0] INFO JobContainer - Running by standalone Mode.
2022-03-23 00:54:03.556 [taskGroup-0] INFO TaskGroupContainer - taskGroupId=[0] start [1] channels for [1] tasks.
2022-03-23 00:54:03.560 [taskGroup-0] INFO Channel - Channel set byte_speed_limit to -1, No bps activated.
2022-03-23 00:54:03.560 [taskGroup-0] INFO Channel - Channel set record_speed_limit to -1, No tps activated.
2022-03-23 00:54:03.576 [taskGroup-0] INFO TaskGroupContainer - taskGroup[0] taskId[0] attemptCount[1] is started
2022-03-23 00:54:03.882 [taskGroup-0] INFO TaskGroupContainer - taskGroup[0] taskId[0] is successed, used[309]ms
2022-03-23 00:54:03.884 [taskGroup-0] INFO TaskGroupContainer - taskGroup[0] completed it's tasks.
2022-03-23 00:54:13.570 [job-0] INFO StandAloneJobContainerCommunicator - Total 100000 records, 2600000 bytes | Speed 253.91KB/s, 10000 records/s | Error 0 records, 0 bytes | All Task WaitWriterTime 0.039s | All Task WaitReaderTime 0.053s | Percentage 100.00%
2022-03-23 00:54:13.570 [job-0] INFO AbstractScheduler - Scheduler accomplished all tasks.
2022-03-23 00:54:13.570 [job-0] INFO JobContainer - DataX Writer.Job [streamwriter] do post work.
2022-03-23 00:54:13.570 [job-0] INFO JobContainer - DataX Reader.Job [streamreader] do post work.
2022-03-23 00:54:13.571 [job-0] INFO JobContainer - DataX jobId [0] completed successfully.
2022-03-23 00:54:13.572 [job-0] INFO HookInvoker - No hook invoked, because base dir not exists or is a file: /opt/datax/hook
2022-03-23 00:54:13.574 [job-0] INFO JobContainer -
[total cpu info] =>
averageCpu | maxDeltaCpu | minDeltaCpu
-1.00% | -1.00% | -1.00%
[total gc info] =>
NAME | totalGCCount | maxDeltaGCCount | minDeltaGCCount | totalGCTime | maxDeltaGCTime | minDeltaGCTime
PS MarkSweep | 0 | 0 | 0 | 0.000s | 0.000s | 0.000s
PS Scavenge | 0 | 0 | 0 | 0.000s | 0.000s | 0.000s
2022-03-23 00:54:13.574 [job-0] INFO JobContainer - PerfTrace not enable!
2022-03-23 00:54:13.574 [job-0] INFO StandAloneJobContainerCommunicator - Total 100000 records, 2600000 bytes | Speed 253.91KB/s, 10000 records/s | Error 0 records, 0 bytes | All Task WaitWriterTime 0.039s | All Task WaitReaderTime 0.053s | Percentage 100.00%
2022-03-23 00:54:13.576 [job-0] INFO JobContainer -
任务启动时刻 : 2022-03-23 00:54:03
任务结束时刻 : 2022-03-23 00:54:13
任务总计耗时 : 10s
任务平均流量 : 253.91KB/s
记录写入速度 : 10000rec/s
读出记录总数 : 100000
读写失败总数 : 0
- 记录datax初次问题
版权声明：本文为liguanhaoyonghu原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接和本声明。