Oozie 集成 Sqoop

1) modify yarn to avoid workflow hang
$ cd ~/work/hadoop/etc/hadoop
$ diff -u yarn-site.xml.orig yarn-site.xml

--- yarn-site.xml.orig	2022-06-21 14:44:21.000000000 +0800
+++ yarn-site.xml	2022-06-29 16:03:34.000000000 +0800
@@ -29,4 +29,8 @@
         <name>yarn.log-aggregation.retain-seconds</name>
         <value>604800</value>
     </property>
+    <property>
+        <name>yarn.resourcemanager.scheduler.class</name>
+        <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
+    </property>
 </configuration>

2) prepare the case
$ cd ~/work/oozie-5.2.1
$ tree oozie/apps/sqoop_import

oozie/apps/sqoop_import
├── job.properties
├── lib
│   └── mysql-connector-java-8.0.28.jar
└── workflow.xm

$ cat oozie/apps/sqoop_import/job.properties

nameNode=hdfs://localhost:9000
resourceManager=localhost:8032
queueName=default
oozieRoot=user/${user.name}/oozie
 
oozie.use.system.libpath=true
 
oozie.wf.application.path=${nameNode}/${oozieRoot}/apps/sqoop_import
outputDir=data/sqoop_import/output

$ cat oozie/apps/sqoop_import/workflow.xml

<workflow-app xmlns="uri:oozie:workflow:1.0" name="sqoop-wf">
    <start to="sqoop-node"/>
 
    <action name="sqoop-node">
        <sqoop xmlns="uri:oozie:sqoop-action:1.0">
            <resource-manager>${resourceManager}</resource-manager>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${nameNode}/${oozieRoot}/${outputDir}"/>
            </prepare>
            <configuration>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
            <arg>import</arg>
            <arg>--connect</arg>
            <arg>jdbc:mysql://localhost:3306/manga</arg>
            <arg>--username</arg>
            <arg>manga</arg>
            <arg>--password</arg>
            <arg>manga</arg>
            <arg>--table</arg>
            <arg>fruit</arg>
            <arg>--target-dir</arg>
            <arg>/${oozieRoot}/${outputDir}</arg>
            <arg>-m</arg>
            <arg>1</arg>
        </sqoop>
        <ok to="end"/>
        <error to="fail"/>
    </action>
 
    <kill name="fail">
        <message>Sqoop failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>

3) upload to hdfs
$ hdfs dfs -put oozie/apps/sqoop_import oozie/apps/

4) run and check
$ bin/oozie job -config oozie/apps/sqoop_import/job.properties -run
job: 0000000-220629212524491-oozie-sun_-W
~/work/oozie-5.2.1$ bin/oozie job -info 0000000-220629212524491-oozie-sun_-W  

Job ID : 0000000-220629212524491-oozie-sun_-W
------------------------------------------------------------------------------------------------------------------------------------
Workflow Name : sqoop-wf
App Path      : hdfs://localhost:9000/user/sun_xo/oozie/apps/sqoop_import
Status        : SUCCEEDED
Run           : 0
User          : sun_xo
Group         : -
Created       : 2022-06-29 13:27 GMT
Started       : 2022-06-29 13:27 GMT
Last Modified : 2022-06-29 13:27 GMT
Ended         : 2022-06-29 13:27 GMT
CoordAction ID: -
 
Actions
------------------------------------------------------------------------------------------------------------------------------------
ID                                                                            Status    Ext ID                 Ext Status Err Code  
------------------------------------------------------------------------------------------------------------------------------------
0000000-220629212524491-oozie-sun_-W@:start:                                  OK        -                      OK         -         
------------------------------------------------------------------------------------------------------------------------------------
0000000-220629212524491-oozie-sun_-W@sqoop-node                               OK        application_1656509094029_0001SUCCEEDED  -         
------------------------------------------------------------------------------------------------------------------------------------
0000000-220629212524491-oozie-sun_-W@end                                      OK        -                      OK         -         
------------------------------------------------------------------------------------------------------------------------------------

Note: Oozie application logs can be found under hdfs://localhost:9000/tmp/${user.name}/logs/,  which is very useful in troubleshooting

...
Oozie Launcher Application Master configuration
===============================================
Workflow job id   : 0000000-220629212524491-oozie-sun_-W
Workflow action id: 0000000-220629212524491-oozie-sun_-W@sqoop-node
...