{"id":167,"date":"2020-11-03T22:16:12","date_gmt":"2020-11-03T13:16:12","guid":{"rendered":"http:\/\/localhost:8000\/?p=167"},"modified":"2021-01-23T12:30:42","modified_gmt":"2021-01-23T03:30:42","slug":"airflow-dockeroperator","status":"publish","type":"post","link":"http:\/\/localhost:8000\/2020\/11\/airflow-dockeroperator.html","title":{"rendered":"Airflow\u3067DockerOperator\u3092\u52d5\u304b\u3059"},"content":{"rendered":"<p><a href=\"https:\/\/airflow.apache.org\/\">Airflow<\/a>\u306f\u3001\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u3092\u30d7\u30ed\u30b0\u30e9\u30e0\u3067\u4f5c\u6210\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3001\u304a\u3088\u3073\u76e3\u8996\u3059\u308b\u305f\u3081\u306e\u30d7\u30e9\u30c3\u30c8\u30d5\u30a9\u30fc\u30e0\u3067\u3059\u3002\u4ed5\u4e8b\u3067Airflow\u3092\u63a1\u7528\u3059\u308b\u3053\u3068\u306b\u3057\u305f\u306e\u3067\u3001\u4eca\u56de\u306fAirflow\u3092\u30ed\u30fc\u30ab\u30ebPC\u4e0a\u3067\u52d5\u304b\u3057\u3066\u3001DockerOperator\u3067\u4efb\u610f\u306eDocker\u30b3\u30f3\u30c6\u30ca\u3092\u5b9f\u884c\u3057\u3064\u3064\u3001\u5fc5\u8981\u306a\u8981\u7d20\uff08DAG\u5b9f\u884c\u6642\u306e\u30d1\u30e9\u30e1\u30fc\u30bf\u6307\u5b9a\u3068\u304b\u74b0\u5883\u5909\u6570\u6e21\u3057\u306a\u3069\uff09\u3092\u8a66\u3057\u3066\u3044\u304d\u305f\u3044\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n<h2>Airflow\u306b\u3064\u3044\u3066<\/h2>\n<p>\u516c\u5f0f\u306e\u30c8\u30c3\u30d7\u30da\u30fc\u30b8\u304b\u3089\u5f15\u3063\u5f35\u3063\u3066\u304d\u305f\u3060\u3051\u3067\u3059\u304c\u3001\u4ee5\u4e0b\u306e\u3088\u3046\u306a\u539f\u5247\u30fb\u7279\u5fb4\u304c\u3042\u308b\u305d\u3046\u3067\u3059\u3002<\/p>\n<h3>\u539f\u5247<\/h3>\n<p><strong>\u30b9\u30b1\u30fc\u30e9\u30d6\u30eb<\/strong><br \/>\nAirflow\u306f\u30e2\u30b8\u30e5\u30fc\u30eb\u5316\u3055\u308c\u305f\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u3092\u5099\u3048\u3066\u3044\u3066\u3001Message\u30ad\u30e5\u30fc\u3092\u4f7f\u7528\u3057\u3066\u4efb\u610f\u306e\u6570\u306e\u30ef\u30fc\u30ab\u30fc\u3092\u8abf\u6574\u3067\u304d\u3001\u7121\u9650\u306b\u62e1\u5f35\u3067\u304d\u308b<\/p>\n<p><strong>\u52d5\u7684<\/strong><br \/>\nAirflow\u306e\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u306fPython\u3067\u5b9a\u7fa9\u3055\u308c\u3066\u304a\u308a\u3001\u52d5\u7684\u306b\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u3092\u751f\u6210\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b<\/p>\n<p><strong>\u62e1\u5f35\u53ef\u80fd<\/strong><br \/>\n\u72ec\u81ea\u306e\u6f14\u7b97\u5b50\u3092\u7c21\u5358\u306b\u5b9a\u7fa9\u3057\u3001\u30e9\u30a4\u30d6\u30e9\u30ea\u3092\u62e1\u5f35\u3057\u3066\u3001\u74b0\u5883\u306b\u9069\u3057\u305f\u62bd\u8c61\u5316\u30ec\u30d9\u30eb\u306b\u9069\u5408\u3055\u305b\u308b\u3053\u3068\u304c\u3067\u304d\u308b<\/p>\n<p><strong>\u30a8\u30ec\u30ac\u30f3\u30c8<\/strong><br \/>\nAirflow\u306e\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u306f\u7121\u99c4\u304c\u306a\u304f\u660e\u78ba\u3067\u3042\u308a\u3001\u30d1\u30e9\u30e1\u30fc\u30bf\u5316\u306fJinja\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u30a8\u30f3\u30b8\u30f3\u3092\u4f7f\u7528\u3057\u3066\u30b3\u30a2\u306b\u7d44\u307f\u8fbc\u307e\u308c\u3066\u3044\u308b<\/p>\n<h3>\u7279\u5fb4<\/h3>\n<p><strong>\u7d14\u7c8b\u306aPython<\/strong><br \/>\n\u30b3\u30de\u30f3\u30c9\u30e9\u30a4\u30f3\u3084XML\u306e\u9ed2\u9b54\u8853\u306f\u5fc5\u8981\u306a\u3057\uff01\u6a19\u6e96\u306ePython\u6a5f\u80fd\u3092\u4f7f\u7528\u3057\u3066\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u306e\u65e5\u6642\u5f62\u5f0f\u3084\u30bf\u30b9\u30af\u3092\u52d5\u7684\u306b\u751f\u6210\u3059\u308b\u30eb\u30fc\u30d7\u306a\u3069\u3092\u5b9f\u88c5\u3057\u3001\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u3092\u5b8c\u5168\u306a\u67d4\u8edf\u306b\u69cb\u7bc9\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b<\/p>\n<p><strong>\u4fbf\u5229\u306aUI<\/strong><br \/>\n\u5805\u7262\u3067\u6700\u65b0\u306eWeb\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u4ecb\u3057\u3066\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u3092\u76e3\u8996\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3001\u304a\u3088\u3073\u7ba1\u7406\u3067\u304d\u308b\u3002\u30bf\u30b9\u30af\u306e\u30b9\u30c6\u30fc\u30bf\u30b9\u3068\u30ed\u30b0\u3092\u5e38\u306b\u5b8c\u5168\u306bWebUI\u304b\u3089\u628a\u63e1\u3067\u304d\u308b<\/p>\n<p><strong>\u5805\u7262\u306a\u7d71\u5408<\/strong><br \/>\nAirflow\u306f\u3001GCP\u3001AWS\u3001Azure\u3001\u304a\u3088\u3073\u305d\u306e\u4ed6\u306e\u591a\u304f\u306e\u30b5\u30fc\u30c9\u30d1\u30fc\u30c6\u30a3\u30b5\u30fc\u30d3\u30b9\u3067\u633f\u5165\u3057\u3066\u5b9f\u884c\u53ef\u80fd\u306a\u30aa\u30da\u30ec\u30fc\u30bf\u30fc\u3092\u5099\u3048\u3066\u3044\u308b\u3002\u3053\u308c\u306b\u3088\u308a\u3001Airflow\u3092\u73fe\u5728\u306e\u30a4\u30f3\u30d5\u30e9\u30b9\u30c8\u30e9\u30af\u30c1\u30e3\u306b\u7c21\u5358\u306b\u9069\u7528\u3057\u3001\u6b21\u4e16\u4ee3\u30c6\u30af\u30ce\u30ed\u30b8\u30fc\u306b\u62e1\u5f35\u3067\u304d\u308b<\/p>\n<p><strong>\u4f7f\u3044\u3084\u3059\u3044<\/strong><br \/>\nPython\u306e\u77e5\u8b58\u304c\u3042\u308c\u3070\u3001\u8ab0\u3067\u3082\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u3092\u30c7\u30d7\u30ed\u30a4\u3067\u304d\u308b\u3002Apache Airflow\u306f\u3001\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u306e\u7bc4\u56f2\u3092\u5236\u9650\u3057\u306a\u3044\u3002\u3053\u308c\u306b\u3088\u308a\u3001ML\u30e2\u30c7\u30eb\u306e\u69cb\u7bc9\u3001\u30c7\u30fc\u30bf\u306e\u8ee2\u9001\u3001\u30a4\u30f3\u30d5\u30e9\u30b9\u30c8\u30e9\u30af\u30c1\u30e3\u306e\u7ba1\u7406\u306a\u3069\u3092\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u306e\u4e2d\u3067\u81ea\u7531\u306b\u5b9a\u7fa9\u30fb\u5b9f\u884c\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b<\/p>\n<p><strong>\u30aa\u30fc\u30d7\u30f3\u30bd\u30fc\u30b9<\/strong><br \/>\n\u6539\u5584\u3057\u305f\u3044\u5834\u5408\u306fIssue\u3092\u3042\u3052\u308b\u3053\u3068\u3082\u3067\u304d\u308b\u3057\u3001Pull Request\u3092\u3059\u308b\u3053\u3068\u3082\u3067\u304d\u308b\u3002\u4f5c\u696d\u306f\u5358\u7d14\u3067\u30cf\u30fc\u30c9\u30eb\u3082\u9577\u6642\u9593\u306e\u624b\u9806\u3082\u5fc5\u8981\u306a\u3057<\/p>\n<h2>\u63a1\u7528\u7406\u7531<\/h2>\n<p>\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u30a8\u30f3\u30b8\u30f3\u306f\u3053\u308c\u307e\u3067 <a href=\"https:\/\/rinoguchi.net\/2019\/10\/jenkins-ubuntu.html\">Jenkins<\/a> \u3092\u5229\u7528\u3057\u3066\u3044\u305f\u306e\u3067\u3059\u304c\u3001\u4f55\u3067\u3082\u3067\u304d\u308b\u53cd\u9762\u8a2d\u5b9a\u304c\u8907\u96d1\u3067\u53e4\u81ed\u3044\u5370\u8c61\u304c\u3042\u308a\u3001\u5225\u306e\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u30a8\u30f3\u30b8\u30f3\u306b\u4e57\u308a\u63db\u3048\u305f\u3044\u3068\u601d\u3063\u3066\u3044\u307e\u3057\u305f\u3002\u4ee5\u4e0b\u306e\u8981\u4ef6\u3067\u4e57\u308a\u63db\u3048\u5148\u3092\u63a2\u3057\u59cb\u3081\u305f\u306e\u3067\u3059\u304c\u3001Airflow\u304c\u5168\u3066\u6761\u4ef6\u3092\u6e80\u305f\u3057\u3066\u3044\u305f\u306e\u3067\u63a1\u7528\u3059\u308b\u3053\u3068\u306b\u3057\u307e\u3057\u305f\u3002<\/p>\n<ul>\n<li>UI\u304b\u3089\u30d1\u30e9\u30e1\u30fc\u30bf\u3092\u6307\u5b9a\u3057\u3066\u30b8\u30e7\u30d6\u3092\u5b9f\u884c\u3067\u304d\u308b\n<ul>\n<li>=&gt; DAG\u5b9f\u884c\u6642\u306bJSON\u3067\u6307\u5b9a\u3067\u304d\u308b\u3002<a href=\"https:\/\/airflow.apache.org\/blog\/airflow-1.10.10\/#allow-passing-dagrun-conf-when-triggering-dags-via-ui\">v1.10.10<\/a> \u3067\u5c0e\u5165\u3055\u308c\u305f<\/li>\n<\/ul>\n<\/li>\n<li>\u30b8\u30e7\u30d6\u5b9f\u884c\u5c65\u6b74\u304a\u3088\u3073\u30ed\u30b0\u3092UI\u3067\u78ba\u8a8d\u3067\u304d\u308b\n<ul>\n<li>=&gt; \u5145\u5b9f\u3057\u305fWebUI\u304c\u63d0\u4f9b\u3055\u308c\u3066\u304a\u308a\u3001\u5c65\u6b74\u3084\u30ed\u30b0\u3092\u898b\u308b\u3050\u3089\u3044\u306f\u4f59\u88d5\u3067\u3067\u304d\u308b<\/li>\n<\/ul>\n<\/li>\n<li>\u30b8\u30e7\u30d6\u3092\u500b\u5225\u306e\u72ec\u7acb\u3057\u305f\u5b9f\u884c\u74b0\u5883\u3067\u5b9f\u884c\u3067\u304d\u308b\n<ul>\n<li>=&gt; DockerOperator\u3092\u4f7f\u3048\u3070\u3067\u304d\u308b<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<h2>\u691c\u8a3c<\/h2>\n<h3>Docker\u3067Airflow\u3092\u52d5\u304b\u3059<\/h3>\n<p>\u672c\u6765\u306f<a href=\"https:\/\/airflow.apache.org\/docs\/stable\/installation.html\">\u3053\u3061\u3089<\/a>\u3092\u5143\u306b\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3057\u305f\u65b9\u304c\u52c9\u5f37\u306b\u306a\u308b\u3068\u601d\u3046\u306e\u3067\u3059\u304c\u3001\u4eca\u56de\u306f\u30b5\u30af\u30c3\u3068Docker\u3067Airflow\u3092\u52d5\u304b\u3057\u3066\u3001\u3055\u3089\u306b\u305d\u306e\u4e2d\u3067DockerOperator\u3092\u4f7f\u3063\u3066\u4efb\u610f\u306e\u30a2\u30d7\u30ea\u30b1\u30fc\u30b7\u30e7\u30f3\u3092\u52d5\u304b\u305b\u308b\u304b\u3069\u3046\u304b\u3092\u691c\u8a3c\u3057\u305f\u3044\u306e\u3067\u3001<a href=\"https:\/\/github.com\/puckel\/docker-airflow\">docker-airflow<\/a>\u3092\u5229\u7528\u3059\u308b\u3053\u3068\u306b\u3057\u307e\u3057\u305f\u3002<\/p>\n<h4>docker\u304a\u3088\u3073docker-compose\u3092\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb<\/h4>\n<p>\u307e\u3060\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3057\u3066\u306a\u3044\u5834\u5408\u306f\u3001\u4ee5\u4e0b\u306e\u30b5\u30a4\u30c8\u3092\u53c2\u8003\u306b\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3057\u307e\u3059\u3002<\/p>\n<ul>\n<li><a href=\"https:\/\/www.docker.com\/get-started\">https:\/\/www.docker.com\/get-started<\/a><\/li>\n<li><a href=\"https:\/\/docs.docker.com\/compose\/install\/\">https:\/\/docs.docker.com\/compose\/install\/<\/a><\/li>\n<\/ul>\n<h4>git clone<\/h4>\n<p>docker-airflow\u3092git clone\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-sh\">git clone https:\/\/github.com\/puckel\/docker-airflow.git\ncd docker-airflow<\/code><\/pre>\n<h4>Docker\u30a4\u30e1\u30fc\u30b8\u3092\u4f5c\u6210<\/h4>\n<p>DockerFile\u3092\u5143\u306b<code>puckel\/docker-airflow:latest<\/code>\u3068\u3044\u3046\u30bf\u30b0\u540d\u3067Docker\u30a4\u30e1\u30fc\u30b8\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-sh\">docker build --rm --build-arg AIRFLOW_DEPS=&quot;datadog,dask&quot; --build-arg PYTHON_DEPS=&quot;flask_oauthlib&gt;=0.9&quot; -t puckel\/docker-airflow .\n> Step 1\/25 : FROM python:3.7-slim-buster\n>  ---&gt; 9703nlsdfgego\n> Step 25\/25 : CMD [&quot;webserver&quot;]\n>  ---&gt; Running in 7sadkdbafsv\n> \u7701\u7565\n> Running in bf7nldasferb\n> Removing intermediate container we4lndfg9fg\n>  ---&gt; 8safdklje4nfgdskjg\n> Successfully built 8safdklje4nfgdskjg\n> Successfully tagged puckel\/docker-airflow:latest<\/code><\/pre>\n<h4>Airflow\u3092\u8d77\u52d5<\/h4>\n<p><a href=\"https:\/\/airflow.apache.org\/docs\/stable\/executor\/index.html\">Executor<\/a>\u3068\u306f\u30bf\u30b9\u30af\u30b9\u30b1\u30b8\u30e5\u30fc\u30e9\u306e\u3053\u3068\u3067\u8272\u3005\u306a\u9078\u629e\u80a2\u304c\u3042\u308b\u3088\u3046\u306a\u306e\u3067\u3059\u304c\u3001\u4eca\u56de\u306f <a href=\"https:\/\/docs.celeryproject.org\/en\/stable\/index.html\">Celery<\/a> \u3092\u5229\u7528\u3059\u308bCeleryExecutor\u3067\u691c\u8a3c\u3092\u9032\u3081\u3088\u3046\u3068\u601d\u3044\u307e\u3059\u3002<br \/>\ndocker-compose\u3067Airflow\u3092\u8d77\u52d5\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-sh\">docker-compose -f docker-compose-CeleryExecutor.yml up -d\n> Starting docker-airflow_webserver_1 ... done\n> Starting docker-airflow_scheduler_1 ... done\n> Starting docker-airflow_flower_1    ... done\n> Starting docker-airflow_worker_1    ... done\n> Starting docker-airflow_postgres_1 ... done\n> Starting docker-airflow_redis_1    ... done<\/code><\/pre>\n<p>\u8d77\u52d5\u3057\u3066\u307f\u308b\u3068\u308f\u304b\u308b\u306e\u3067\u3059\u304c\u3001\u8272\u3005\u306a\u30b3\u30f3\u30c6\u30ca\u304c\u8d77\u52d5\u3057\u307e\u3059\u3002\u4e0a\u304b\u3089\u9806\u306b\u8efd\u304f\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n<ul>\n<li>webserver =&gt; Airflow\u306eWebUI\u3002\u3053\u3053\u304b\u3089DAG\u3092\u5b9f\u884c\u3057\u305f\u308a\u5b9f\u884c\u7d50\u679c\u3092\u78ba\u8a8d\u3057\u305f\u308a\u3059\u308b\u3002<a href=\"https:\/\/flask.palletsprojects.com\/en\/1.1.x\/\">Flask<\/a> \u3067\u5b9f\u88c5\u3055\u308c\u3066\u3044\u308b\u3002<\/li>\n<li>scheduler =&gt; \u30bf\u30b9\u30af\u30b1\u30b8\u30e5\u30fc\u30e9\u306eCelery\u3092\u8d77\u52d5<\/li>\n<li>flower =&gt; Celery\u3092\u30e2\u30cb\u30bf\u30ea\u30f3\u30b0\u3059\u308b\u305f\u3081\u306eWebUI\u3067\u3042\u308b<a href=\"https:\/\/flower.readthedocs.io\/en\/latest\/\">Flower<\/a> \u3092\u8d77\u52d5<\/li>\n<li>worker =&gt; \u30bf\u30b9\u30af\u304c\u5b9f\u884c\u3055\u308c\u308b<\/li>\n<li>postgres =&gt; \u30b8\u30e7\u30d6\u7ba1\u7406\u7528\u306e\u5404\u7a2e\u30c7\u30fc\u30bf\u3092\u7ba1\u7406\u3059\u308b<\/li>\n<li>redis =&gt; \u30d6\u30ed\u30fc\u30ab\u30fc\uff08\u30e1\u30c3\u30bb\u30fc\u30b8\u30ad\u30e5\u30fc\uff09\u3068\u3057\u3066\u3001scheduler\u3068worker\u3092\u4ef2\u4ecb\u3059\u308b<\/li>\n<\/ul>\n<h4>Web UI\u8868\u793a<\/h4>\n<p>\u4ee5\u4e0b\u306e\u753b\u9762\u304c\u8868\u793a\u3055\u308c\u308c\u3070\u3053\u3053\u307e\u3067\u306f\u554f\u984c\u306a\u304f\u52d5\u3044\u3066\u3044\u307e\u3059<\/p>\n<ul>\n<li>airflow\n<ul>\n<li><a href=\"http:\/\/localhost:8080\">http:\/\/localhost:8080<\/a><\/li>\n<\/ul>\n<\/li>\n<li>flower\n<ul>\n<li><a href=\"http:\/\/localhost:5555\">http:\/\/localhost:5555<\/a><\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<h4>\u30c1\u30e5\u30fc\u30c8\u30ea\u30a2\u30ebDAG\u3092\u5b9f\u884c<\/h4>\n<p><a href=\"http:\/\/localhost:8080\">http:\/\/localhost:8080<\/a> \u3067\u958b\u304fAirflow\u306e\u30c8\u30c3\u30d7\u30da\u30fc\u30b8\u306b\u306f\u8aad\u307f\u8fbc\u3093\u3060DAG\u306e\u4e00\u89a7\u304c\u8868\u793a\u3055\u308c\u307e\u3059\u3002<br \/>\n\u30c7\u30d5\u30a9\u30eb\u30c8\u3067turorial\uff08tuto.py\uff09\u304c\u8868\u793a\u3055\u308c\u3066\u3044\u3066\u3044\u307e\u3059\u304c\u3001Off=&gt;On\u306btoggle\u3059\u308b\u3053\u3068\u3067DAG\u304c\u6709\u52b9\u5316\u3055\u308c\u3001\u904e\u53bb\u306b\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3055\u308c\u305fDAG\uff08\u30bf\u30b9\u30af\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\uff09\u304c\u307e\u3068\u3081\u3066\u5b9f\u884c\u3055\u308c\u307e\u3059\u3002<br \/>\n<img src=\"http:\/\/localhost:8000\/wp-content\/uploads\/2021\/01\/20201101174443-600x221.png\" alt=\"\" \/><\/p>\n<p>DAG\u540d\u306e\u30ea\u30f3\u30af\u3092\u30af\u30ea\u30c3\u30af\u3059\u308b\u3068\u8a73\u7d30\u753b\u9762\uff08Tree View\uff09\u306b\u9077\u79fb\u3057\u307e\u3059\u3002\u3053\u3053\u3067\u306fDAG\u306e\u5b9a\u7fa9\u304cTree\u5f62\u5f0f\u3067\u5de6\u5074\u306b\u8868\u793a\u3055\u308c\u3001\u53f3\u5074\u306bDAG\u306e\u5b9f\u884c\u5c65\u6b74\u304c\u8868\u793a\u3055\u308c\u307e\u3059\u3002\u7e26\u3067\u4e00\u3064\u306eDAG\u5b9f\u884c\u306b\u306a\u308a\u307e\u3059\u3002<br \/>\n<img src=\"http:\/\/localhost:8000\/wp-content\/uploads\/2021\/01\/20201101175816-600x203.png\" alt=\"\" \/><\/p>\n<p>\u25a0\u304c\u4e00\u3064\u306e\u30bf\u30b9\u30af\u306b\u306a\u3063\u3066\u3044\u3066\u3001\u30af\u30ea\u30c3\u30af\u3059\u308b\u3068\u4ee5\u4e0b\u306e\u753b\u9762\u304c\u8868\u793a\u3055\u308c\u307e\u3059\u3002<br \/>\n<img src=\"http:\/\/localhost:8000\/wp-content\/uploads\/2021\/01\/20201101180140-600x485.png\" alt=\"\" \/><\/p>\n<p>\u3053\u3061\u3089\u304b\u3089\u3001\u30ed\u30b0\u3092\u8868\u793a\u3057\u305f\u308a\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<br \/>\n<img src=\"http:\/\/localhost:8000\/wp-content\/uploads\/2021\/01\/20201101180538-600x387.png\" alt=\"\" \/><\/p>\n<p>\u3042\u3068\u306f<code>Trigger DAG<\/code>\u304b\u3089DAG\u3092\u624b\u52d5\u3067\u5b9f\u884c\u3059\u308b\u3053\u3068\u3082\u3067\u304d\u307e\u3059\u3002<\/p>\n<h4>\u30c1\u30e5\u30fc\u30c8\u30ea\u30a2\u30ebDAG\u30b3\u30fc\u30c9\u3092\u78ba\u8a8d<\/h4>\n<p>DAG\u306e\u30b3\u30fc\u30c9\u306fCode\u30dc\u30bf\u30f3\u304b\u3089\u78ba\u8a8d\u3067\u304d\u307e\u3059\u3002\u30d7\u30ed\u30b8\u30a7\u30af\u30c8\u914d\u4e0b\u306e<code>dags\/tuto.py<\/code>\u3068\u540c\u4e00\u3067\u3059\u3002<br \/>\n<img src=\"http:\/\/localhost:8000\/wp-content\/uploads\/2021\/01\/20201101180927-600x237.png\" alt=\"\" \/><\/p>\n<p>\u30b3\u30fc\u30c9\u5168\u6587\u3067\u3059\u3002<\/p>\n<pre><code class=\"language-python\">from airflow import DAG\nfrom airflow.operators.bash_operator import BashOperator\nfrom datetime import datetime, timedelta\n\ndefault_args = {\n    &quot;owner&quot;: &quot;airflow&quot;,\n    &quot;depends_on_past&quot;: False,\n    &quot;start_date&quot;: datetime(2015, 6, 1),\n    &quot;email&quot;: [&quot;airflow@airflow.com&quot;],\n    &quot;email_on_failure&quot;: False,\n    &quot;email_on_retry&quot;: False,\n    &quot;retries&quot;: 1,\n    &quot;retry_delay&quot;: timedelta(minutes=5),\n}\n\ndag = DAG(&quot;tutorial&quot;, default_args=default_args, schedule_interval=timedelta(1))\n\n## t1, t2 and t3 are examples of tasks created by instantiating operators\nt1 = BashOperator(task_id=&quot;print_date&quot;, bash_command=&quot;date&quot;, dag=dag)\n\nt2 = BashOperator(task_id=&quot;sleep&quot;, bash_command=&quot;sleep 5&quot;, retries=3, dag=dag)\n\ntemplated_command = &quot;&quot;&quot;\n    {% for i in range(5) %}\n        echo &quot;{{ ds }}&quot;\n        echo &quot;{{ macros.ds_add(ds, 7)}}&quot;\n        echo &quot;{{ params.my_param }}&quot;\n    {% endfor %}\n&quot;&quot;&quot;\n\nt3 = BashOperator(\n    task_id=&quot;templated&quot;,\n    bash_command=templated_command,\n    params={&quot;my_param&quot;: &quot;Parameter I passed in&quot;},\n    dag=dag,\n)\n\nt2.set_upstream(t1)\nt3.set_upstream(t1)<\/code><\/pre>\n<ul>\n<li><code>default_args<\/code>\u3067DAG\u30ec\u30d9\u30eb\u3067\u306e\u30aa\u30da\u30ec\u30fc\u30bf\u306e\u30d1\u30e9\u30e1\u30fc\u30bf\u3092\u5b9a\u7fa9\u3057\u307e\u3059\u3002\u5b9a\u7fa9\u3067\u304d\u308b\u5024\u306f<a href=\"https:\/\/airflow.apache.org\/docs\/stable\/_api\/airflow\/models\/index.html?highlight=dag#airflow.models.BaseOperator\">BaseOperator\u306e\u30d1\u30e9\u30e1\u30fc\u30bf<\/a> \u3067\u3059\u3002\u3053\u3061\u3089\u306f\u3042\u304f\u307e\u3067\u30c7\u30d5\u30a9\u30eb\u30c8\u5024\u306a\u306e\u3067Operator\u30ec\u30d9\u30eb\u3067\u4e0a\u66f8\u304d\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/li>\n<li><code>DAG(&quot;tutorial&quot;, schedule_interval=timedelta(1))<\/code>\u30671\u65e5\u6bce\u306b\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u5b9f\u884c\u3059\u308b\u8a2d\u5b9a\u306b\u306a\u3063\u3066\u3044\u308b<\/li>\n<li><code>t1<\/code>, <code>t2<\/code>, <code>t3<\/code>\u306f\u5168\u3066BashOperator\u3092\u5229\u7528\u3057\u3066\u3044\u308b\n<ul>\n<li>BashOperator\u306fwoker\u30b3\u30f3\u30c6\u30ca\u306elinux\u30b5\u30fc\u30d0\u5185\u3067\u5b9f\u884c\u3055\u308c\u308b\u305f\u3081\u3001\u5fc5\u8981\u306alinux\u30d1\u30c3\u30b1\u30fc\u30b8\u306f\u3001DockerFile\u3067<code>apt get<\/code>\u306a\u3069\u3067\u4e8b\u524d\u306b\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3059\u308b<\/li>\n<\/ul>\n<\/li>\n<li><code>t2.set_upstream(t1)<\/code>\u3001<code>t3.set_upstream(t1)<\/code>\u306f\u3001<code>t2<\/code>\u3068<code>t3<\/code>\u306e\u4e0a\u6d41\u304c<code>t1<\/code>\u3068\u3044\u3046\u8a2d\u5b9a\u3060\u304c\u5c11\u3057\u308f\u304b\u308a\u306b\u304f\u3044\n<ul>\n<li><code>&gt;&gt;<\/code>\u3092\u4f7f\u3063\u3066\u5b9f\u88c5\u3067\u304d\u308b\u3002\u4ee5\u4e0b\u306e\u3069\u3061\u3089\u3067\u3082OK<\/li>\n<li><code>t1 &gt;&gt; (t2, t3)<\/code><\/li>\n<li><code>t1 &gt;&gt; [t2, t3]<\/code><\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<h3>DockerOperator\u3092\u52d5\u304b\u3059<\/h3>\n<p>\u3068\u308a\u3042\u3048\u305a\u3001tutorial\u306eDAG\u3092\u5b9f\u884c\u3057\u5185\u5bb9\u3082\u3060\u3044\u305f\u3044\u628a\u63e1\u3067\u304d\u305f\u6c17\u304c\u3059\u308b\u306e\u3067\u3001\u6b21\u306f\u672c\u547d\u306eDockerOperator\u3092\u8a66\u3057\u3066\u3044\u304d\u305f\u3044\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n<h4>DAG\u3092\u4f5c\u6210<\/h4>\n<p><code>\/dags<\/code>\u914d\u4e0b\u306b<code>docker_bash.py<\/code>\u3092\u4f5c\u308a\u307e\u3059\u3002\u3053\u308c\u306f\u3001DockerOperator\u3067<code>centos:latest<\/code>\u30a4\u30e1\u30fc\u30b8\u3092\u4f7f\u3063\u3066\u3001bash\u30b3\u30de\u30f3\u30c9\uff0830\u79d2wait\u3059\u308b\uff09\u3092\u5b9f\u884c\u3059\u308bDAG\u3067\u3059\u3002<\/p>\n<p>\u5185\u5bb9\u306f\u898b\u308c\u3070\u5206\u304b\u308b\u69d8\u306a\u5185\u5bb9\u3060\u3068\u601d\u3044\u307e\u3059\u3002\u4e00\u70b9\u88dc\u8db3\u3059\u308b\u3068\u3001<code>docker_url=&quot;unix:\/\/var\/run\/docker.sock&quot;<\/code>\u306f\u30c7\u30d5\u30a9\u30eb\u30c8\u5024\u305d\u306e\u307e\u307e\u306a\u306e\u306b\u3042\u3048\u3066\u66f8\u3044\u3066\u307e\u3059\u3002\u3053\u308c\u306f\u3001Docker\u30c7\u30fc\u30e2\u30f3\u306eUnix\u30bd\u30b1\u30c3\u30c8\u30d1\u30b9\u3067\u3001\u3053\u308c\u3092\u901a\u3058\u3066Docker\u30b3\u30f3\u30c6\u30ca\u306e\u8d77\u52d5\u306a\u3069\u3092\u884c\u3044\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-python\">from airflow import DAG\nfrom airflow.operators.bash_operator import BashOperator\nfrom airflow.operators.docker_operator import DockerOperator\nfrom datetime import datetime, timedelta\n\ndefault_args = {\n    &quot;owner&quot;: &quot;airflow&quot;,\n    &quot;depends_on_past&quot;: False,\n    &quot;start_date&quot;: datetime(2020, 11, 1),\n    &quot;email&quot;: [&quot;airflow@airflow.com&quot;],\n    &quot;email_on_failure&quot;: False,\n    &quot;email_on_retry&quot;: False,\n    &quot;retries&quot;: 1,\n    &quot;retry_delay&quot;: timedelta(minutes=5),\n}\n\nwith DAG(&quot;docker_bash_sample&quot;, default_args=default_args, schedule_interval=timedelta(1)) as dag:\n    t1 = BashOperator(task_id=&quot;print_start&quot;, bash_command=&quot;echo $(date &#039;+%Y-%m-%d %H:%M:%S&#039;) started.&quot;)\n\n    t2 = DockerOperator(\n        task_id=&#039;docker_command&#039;,\n        image=&#039;centos:latest&#039;,\n        api_version=&#039;auto&#039;,\n        auto_remove=True,\n        command=&quot;&quot;&quot;\/bin\/bash -c \\&#039;echo &quot;1&quot; &amp;&amp; sleep 30 &amp;&amp; echo &quot;2&quot;\\&#039;&quot;&quot;&quot;,\n        docker_url=&quot;unix:\/\/var\/run\/docker.sock&quot;,\n        network_mode=&quot;bridge&quot;,\n    )\n\n    t3 = BashOperator(task_id=&quot;print_finish&quot;, bash_command=&quot;echo $(date &#039;+%Y-%m-%d %H:%M:%S&#039;) finished.&quot;)\n    t1 &gt;&gt; t2 &gt;&gt; t3<\/code><\/pre>\n<h4>\u30c8\u30e9\u30d6\u30eb\u5bfe\u5fdc<\/h4>\n<p>Airflow\u306eWebUI\u3092\u30d6\u30e9\u30a6\u30b6\u3067reload\u3059\u308b\u3060\u3051\u3067\u65b0\u3057\u3044DAG\u304c\u8868\u793a\u3055\u308c\u307e\u3059\u306e\u3067\u3001\u305d\u3053\u304b\u3089DAG\u3092\u5b9f\u884c\u3057\u3066\u30b5\u30af\u30c3\u3068\u52d5\u3051\u3070\u3044\u3044\u306e\u3067\u3059\u304c\u3001\u8272\u3005\u3068\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3059\u306e\u3067\u3001\u5bfe\u5fdc\u3057\u3066\u3044\u304f\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n<h5>docker\u30d1\u30c3\u30b1\u30fc\u30b8\u304c\u898b\u3064\u304b\u3089\u306a\u3044<\/h5>\n<blockquote>\n<p>Broken DAG: [\/usr\/local\/airflow\/dags\/docker_bash.py] No module named 'docker'<\/p>\n<\/blockquote>\n<p>Dockerfile\u3067python\u306e<code>docker<\/code>\u30d1\u30c3\u30b1\u30fc\u30b8\u3092\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3057\u307e\u3059\u3002<\/p>\n<pre><code>RUN pip install docker<\/code><\/pre>\n<p>\u3055\u3089\u306b\u3001<code>docker-compose-CeleryExecutor.yml<\/code>\u3092\u4fee\u6b63\u3057\u307e\u3059\u3002\u5b9f\u306f\u3053\u3053\u307e\u3067\u306f\u305b\u3063\u304b\u304f\u4f5c\u3063\u305fAirflow\u306eDocker\u30a4\u30e1\u30fc\u30b8<code>puckel\/docker-airflow:latest<\/code>\u3067\u306f\u306a\u304f\u3001Docker Registry\u304b\u3089\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9\u3057\u305f<code>puckel\/docker-airflow:1.10.9<\/code>\u3092\u5229\u7528\u3057\u3066\u3044\u307e\u3057\u305f\u3002\u307e\u305a\u306f\u305d\u306e\u90e8\u5206\u3092\u4fee\u6b63\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-yaml\">-        image: puckel\/docker-airflow:1.10.9\n+        image: puckel\/docker-airflow:latest<\/code><\/pre>\n<p>\u3064\u3044\u3067\u306b\u3001apache-airflow\u306e\u30d0\u30fc\u30b8\u30e7\u30f3\u304c<code>1.10.9<\/code>\u3068\u5c11\u3057\u53e4\u3044\u306e\u3067\u3001<code>1.10.12<\/code>\u306b\u3042\u3052\u3066\u304a\u304d\u307e\u3059\u3002Dockerfile\u306e\u4ee5\u4e0b\u306e\u90e8\u5206\u3092\u4fee\u6b63\u3057\u307e\u3059\u3002<\/p>\n<pre><code>-ARG AIRFLOW_VERSION=1.10.9\n+ARG AIRFLOW_VERSION=1.10.12<\/code><\/pre>\n<h5>SQLAlchemy\u3067\u30a8\u30e9\u30fc<\/h5>\n<blockquote>\n<p>sqlalchemy.exc.NoInspectionAvailable: No inspection system is available for object of type &lt;class 'method'&gt;<\/p>\n<\/blockquote>\n<p><a href=\"https:\/\/github.com\/puckel\/docker-airflow\/issues\/535\">\u3053\u3061\u3089<\/a>\u3067issue\u304c\u4e0a\u304c\u3063\u3066\u3044\u308b\u306e\u3067\u3059\u304c\u30012020\u5e7411\u6708\u6642\u70b9\u3060\u3068SQLAlchemy\u306e\u6700\u65b0\u30d0\u30fc\u30b8\u30e7\u30f3\u306b\u306f\u5c11\u3057\u554f\u984c\u304c\u3042\u308b\u3088\u3046\u3067\u3059\u3002\u3044\u305a\u308c\u89e3\u6c7a\u3055\u308c\u308b\u3068\u601d\u3044\u307e\u3059\u304c\u3001\u73fe\u72b6\u306b\u304a\u3044\u3066\u306fDockerfile\u306b\u4ee5\u4e0b\u3092\u8ffd\u52a0\u3057\u3066\u56de\u907f\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n<pre><code>RUN pip install &#039;SQLAlchemy==1.3.15&#039;<\/code><\/pre>\n<h5>cattr\u304c\u898b\u3064\u304b\u3089\u306a\u3044<\/h5>\n<blockquote>\n<p>ImportError: cannot import name 'resolve_types' from 'attr' (\/usr\/local\/lib\/python3.7\/site-packages\/attr\/<strong>init<\/strong>.py)<\/p>\n<\/blockquote>\n<p>\u3053\u308c\u306f<code>cattr<\/code>\u3068\u3044\u3046\u30d1\u30c3\u30b1\u30fc\u30b8\u304c\u5b58\u5728\u3057\u306a\u3044\u305f\u3081\u306b\u304a\u3053\u3063\u3066\u3044\u308b\u69d8\u306a\u306e\u3067\u3059\u304c\u3001\u3053\u308c\u3082\u304a\u305d\u3089\u304f\u4e00\u6642\u7684\u306a\u3082\u306e\u3067\u3059\u3002<a href=\"https:\/\/github.com\/apache\/airflow\/issues\/11965\">\u3053\u3061\u3089<\/a>\u306b\u66f8\u3044\u3066\u3042\u308b\u69d8\u306b\u3001Dockerfile\u306b\u5bfe\u5fdc\u3092\u5165\u308c\u307e\u3057\u305f\u3002<\/p>\n<pre><code>RUN pip install apache-airflow==1.10.12 \\\n --constraint &quot;https:\/\/raw.githubusercontent.com\/apache\/airflow\/constraints-1.10.12\/constraints-3.7.txt&quot;<\/code><\/pre>\n<h5>\u30b3\u30f3\u30c6\u30ca\u5185\u304b\u3089\u30db\u30b9\u30c8\u30de\u30b7\u30f3\u306eDocker\u30c7\u30fc\u30e2\u30f3\u306b\u30a2\u30af\u30bb\u30b9\u3067\u304d\u306a\u3044<\/h5>\n<blockquote>\n<p>{{taskinstance.py:1128}} ERROR - Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))<\/p>\n<\/blockquote>\n<p>DockerOperator\u306e\u5f15\u6570\u3067\u6307\u5b9a\u3057\u305f\u3001Docker\u30c7\u30fc\u30e2\u30f3\u306eUnix\u30bd\u30b1\u30c3\u30c8\u30d1\u30b9<code>docker_url=&quot;unix:\/\/var\/run\/docker.sock&quot;<\/code>\u304c\u898b\u3064\u304b\u3089\u306a\u3044\u3068\u3044\u3046\u30a8\u30e9\u30fc\u3067\u3057\u305f\u3002<a href=\"https:\/\/github.com\/puckel\/docker-airflow\/issues\/86\">\u3053\u3061\u3089<\/a>\u3092\u53c2\u8003\u306b\u5bfe\u5fdc\u3057\u3066\u3044\u304d\u307e\u3059\u3002<\/p>\n<p>\u307e\u305a\u3001<code>docker-compose-CeleryExecutor.yml<\/code>\u306eworker\u30b3\u30f3\u30c6\u30ca\u306e\u8a2d\u5b9a\u3092\u8ffd\u52a0\u3057\u307e\u3059\u3002\u3053\u308c\u3067\u30db\u30b9\u30c8\u30de\u30b7\u30f3\u306eDocker\u30c7\u30fc\u30e2\u30f3\u306eUnix\u30bd\u30b1\u30c3\u30c8\u30d1\u30b9\u304c\u30de\u30a6\u30f3\u30c8\u3055\u308c\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-yaml\">     worker:\n         volumes:\n             - .\/dags:\/usr\/local\/airflow\/dags\n+            - \/var\/run\/docker.sock:\/var\/run\/docker.sock<\/code><\/pre>\n<p>\u3053\u306e\u72b6\u614b\u3060\u3068\u307e\u3060\u6a29\u9650\u304c\u8db3\u308a\u305a\u3001\u4ee5\u4e0b\u306e\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3059\u3002<\/p>\n<blockquote>\n<p>{{taskinstance.py:1128}} ERROR - Error while fetching server API version: ('Connection aborted.', PermissionError(13, 'Permission denied'))<\/p>\n<\/blockquote>\n<p>\u307e\u305aDockerfile\u3092\u4fee\u6b63\u3057\u3066\u3001airflow\u30e6\u30fc\u30b6\u306bsudo\u6a29\u9650\u3092\u4e0e\u3048\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-sh\">RUN apt-get update &amp;&amp; \\\n    apt-get -y install sudo\nRUN echo airflow:airflow | chpasswd &amp;&amp; adduser airflow sudo<\/code><\/pre>\n<p>\u3055\u3089\u306b<code>entrypoint.sh<\/code>\u3092\u4fee\u6b63\u3057\u3001\u30b3\u30f3\u30c6\u30ca\u8d77\u52d5\u6642\u306b<code>\/var\/run\/docker.sock<\/code>\u306e\u30d1\u30fc\u30df\u30c3\u30b7\u30e7\u30f3\u3092\u5909\u66f4\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-sh\">if [ -e \/var\/run\/docker.sock ]; then \n  echo airflow | sudo -S chmod 777 \/var\/run\/docker.sock; \nfi<\/code><\/pre>\n<h4>DAG\u3092\u5b9f\u884c<\/h4>\n<p>\u3053\u3053\u307e\u3067\u3067\u3084\u3063\u3068DAG\u304c\u6b63\u5e38\u306b\u5b9f\u884c\u3055\u308c\u307e\u3059\u3002\u5b9f\u884c\u30ed\u30b0\u306f\u3053\u3093\u306a\u611f\u3058\u306b\u306a\u308a\u307e\u3057\u305f\u3002\u8d64\u67a0\u306e\u3068\u3053\u308d\u3092\u898b\u308b\u3068\u300130\u79d2\u30b9\u30ea\u30fc\u30d7\u3057\u3066<code>INFO - 2<\/code>\u3068\u3044\u3046\u30ed\u30b0\u304c\u51fa\u529b\u3055\u308c\u3066\u3044\u308b\u3053\u3068\u304c\u5206\u304b\u308b\u3068\u601d\u3044\u307e\u3059\u3002<br \/>\n<img src=\"http:\/\/localhost:8000\/wp-content\/uploads\/2021\/01\/20201103171733-600x306.png\" alt=\"\" \/><\/p>\n<p>\u53c2\u8003\u307e\u3067\u3067\u3059\u304c\u3001\u6b8b\u5ff5\u306a\u3053\u3068\u306b\u30b3\u30f3\u30c6\u30ca\u5185\u306e\u30ed\u30b0\u304c\u6700\u521d\u306e\u6570\u884c\uff1f\u3060\u3051\u51fa\u529b\u3055\u308c\u307e\u305b\u3093\u3002\u4e0a\u306e\u30ad\u30e3\u30d7\u30c1\u30e3\u3067\u3082\u672c\u6765<code>INFO - 1<\/code>\u3068\u51fa\u529b\u3055\u308c\u3066\u6b32\u3057\u3044\u306e\u3067\u3059\u304c\u51fa\u529b\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u3068\u306f\u3044\u3048\u3001\u5f71\u97ff\u3082\u5c11\u306a\u3044\u3067\u3059\u3057\u3001<a href=\"https:\/\/github.com\/apache\/airflow\/issues\/9190\">ISSUE<\/a>\u3082\u4e0a\u304c\u3063\u3066\u3044\u308b\u307f\u305f\u3044\u306a\u306e\u3067\u3044\u305a\u308c\u6539\u5584\u3055\u308c\u308b\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n<h3>\u81ea\u5206\u3067\u4f5c\u3063\u305fDocker\u30a4\u30e1\u30fc\u30b8\u3092DockerOperator\u3067\u52d5\u304b\u3059<\/h3>\n<p>\u3053\u3053\u306f\u691c\u8a3c\u305b\u305a\u3068\u3082\u52d5\u304f\u306f\u305a\u3067\u3059\u304c\u3001python\u304b\u3089print\u3057\u305f\u30ed\u30b0\u304c\u3061\u3083\u3093\u3068\u51fa\u529b\u3055\u308c\u308b\u304b\u3001\u306a\u3069\u5c11\u3057\u6c17\u306b\u306a\u308b\u306e\u3067\u3084\u3063\u3066\u304a\u304d\u307e\u3059\u3002<\/p>\n<h4>Docker\u30a4\u30e1\u30fc\u30b8\u3092\u4f5c\u6210<\/h4>\n<p><code>\/docker_python_sample<\/code>\u3068\u3044\u3046\u30d5\u30a9\u30eb\u30c0\u3092\u4f5c\u3063\u3066\u305d\u306e\u306a\u304b\u306bDockerfile\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002entrypoint\u3067python\u3092\u5b9f\u884c\u3057\u3066print\u3059\u308b\u3060\u3051\u306e\u8d85\u30b7\u30f3\u30d7\u30eb\u306a\u5185\u5bb9\u3067\u3059\u3002<\/p>\n<pre><code>FROM python:latest\nENTRYPOINT [ &quot;python&quot;, &quot;-c&quot;, &quot;print(&#039;python executed.&#039;)&quot; ]<\/code><\/pre>\n<p>\u4e0a\u8a18\u306eDockerfile\u304b\u3089Docker\u30a4\u30e1\u30fc\u30b8\u3092\u4f5c\u308a\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-sh\">docker build -t python-sample:latest .\/docker_python_sample<\/code><\/pre>\n<h4>DAG\u3092\u4f5c\u6210<\/h4>\n<p><code>\/dags<\/code>\u914d\u4e0b\u306b<code>docker_python.py<\/code>\u3092\u4f5c\u308a\u307e\u3057\u305f\u3002\u5185\u5bb9\u306f<code>docker_bash.py<\/code>\u3068\u307b\u307c\u540c\u3058\u3067\u3001DockerOperator\u306e<code>image<\/code>\u30d1\u30e9\u30e1\u30fc\u30bf\u3092\u5909\u66f4\u3057\u305f\u3060\u3051\u3067\u3059\u3002<\/p>\n<pre><code class=\"language-python\">    t2 = DockerOperator(\n        image=&#039;python_sample:latest&#039;,\n    )<\/code><\/pre>\n<h4>DAG\u3092\u5b9f\u884c<\/h4>\n<p>DAG\u3092trigger\u3057\u305f\u3068\u3053\u308d\u554f\u984c\u306a\u304f\u5b9f\u884c\u3055\u308c\u307e\u3057\u305f\u3002\u5c11\u3057\u61f8\u5ff5\u3057\u3066\u3044\u305f\u30ed\u30b0\u3082\u554f\u984c\u306a\u304f\u51fa\u529b\u3055\u308c\u3066\u3044\u308b\u69d8\u3067\u3059\u3002<br \/>\n<img src=\"http:\/\/localhost:8000\/wp-content\/uploads\/2021\/01\/20201103172955-600x301.png\" alt=\"\" \/><\/p>\n<h3>\u74b0\u5883\u5909\u6570\u3092\u8aad\u307f\u8fbc\u3080<\/h3>\n<p>Docker\u30b3\u30f3\u30c6\u30ca\u3092\u8d77\u52d5\u3059\u308b\u969b\u3001\u74b0\u5883\u5909\u6570\u30d5\u30a1\u30a4\u30eb\uff08.env\u306a\u3069\uff09\u304b\u3089\u74b0\u5883\u5909\u6570\u3092\u8aad\u307f\u8fbc\u3080\u3053\u3068\u304c\u591a\u3044\u3068\u601d\u3044\u307e\u3059\u304c\u3001DockerOperator\u306b\u306f\u74b0\u5883\u5909\u6570\u3092\u8aad\u307f\u8fbc\u3080\u6a5f\u80fd\u306f\u306a\u3044\u307f\u305f\u3044\u3067\u3059\u3002\u3057\u304b\u3057\u3001<code>environment<\/code>\u30d1\u30e9\u30e1\u30fc\u30bf\u306bdict\u5f62\u5f0f\u3067\u74b0\u5883\u5909\u6570\u3092\u6e21\u3059\u3053\u3068\u306f\u3067\u304d\u307e\u3059\u3002\u306a\u306e\u3067\u3001python-dotenv\u3092\u4f7f\u3063\u3066\u74b0\u5883\u5909\u6570\u30d5\u30a1\u30a4\u30eb\u3092dict\u306b\u8aad\u307f\u8fbc\u3093\u3067<code>environment<\/code>\u30d1\u30e9\u30e1\u30fc\u30bf\u306b\u6e21\u3059\u3088\u3046\u306b\u3057\u307e\u3057\u305f\u3002<\/p>\n<p>\u307e\u305a\u306f\u3001Dockerfile\u306bpython-dotenv\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u51e6\u7406\u3092\u8ffd\u52a0\u3057\u307e\u3059\u3002<\/p>\n<pre><code>RUN pip install python-dotenv<\/code><\/pre>\n<p>\u6b21\u306b\u3001docker-compose-CeleryExecutor.yml\u306b\u74b0\u5883\u5909\u6570\u30d5\u30a1\u30a4\u30eb\u306e\u30de\u30a6\u30f3\u30c8\u8a2d\u5b9a\u3092\u8ffd\u52a0\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-yaml\">    worker:\n        volumes:\n            - .\/envfiles:\/usr\/local\/airflow\/envfiles<\/code><\/pre>\n<p>\u6700\u5f8c\u306b\u3001DockerOperator\u3092\u5b9f\u884c\u3059\u308b\u7b87\u6240\u3067\u3001\u74b0\u5883\u5909\u6570\u30d5\u30a1\u30a4\u30eb\u3092\u8aad\u307f\u8fbc\u3093\u3067\u6e21\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-python\">from dotenv import dotenv_values\nDockerOperator(\n    image=&quot;python:latest&quot;,\n    environment=dotenv_values(dotenv_path=&#039;\/usr\/local\/airflow\/envfiles\/sample.env&#039;),  #  \u74b0\u5883\u5909\u6570\u8aad\u307f\u8fbc\u307f\n    command=&quot;&quot;&quot;python -c &quot;from os import environ; print(environ.get(&#039;KEY1&#039;)); print(environ.get(&#039;KEY2&#039;));&quot; &quot;&quot;&quot;  #  \u74b0\u5883\u5909\u6570\u3092\u30ed\u30b0\u51fa\u529b\n)<\/code><\/pre>\n<p>\u554f\u984c\u306a\u304f\u74b0\u5883\u5909\u6570\u304c\u30ed\u30fc\u30c9\u3055\u308c\u3066\u3044\u308b\u3053\u3068\u3092\u30ed\u30b0\u3067\u78ba\u8a8d\u3067\u304d\u307e\u3057\u305f\u3002<\/p>\n<pre><code class=\"language-sh\">[2020-11-07 01:51:09,385] {{docker_operator.py:210}} INFO - Starting docker container from image python:latest\n[2020-11-07 01:51:09,912] {{docker_operator.py:243}} INFO - VALUE1  # &lt;= \u74b0\u5883\u5909\u6570\u306e\u5024\u304c\u30ed\u30b0\u51fa\u529b\u3055\u308c\u3066\u3044\u308b\nVALUE2  # &lt;= \u74b0\u5883\u5909\u6570\u306e\u5024\u304c\u30ed\u30b0\u51fa\u529b\u3055\u308c\u3066\u3044\u308b<\/code><\/pre>\n<h3>\u30af\u30ec\u30c7\u30f3\u30b7\u30e3\u30eb\u30d5\u30a1\u30a4\u30eb\u3092\u8aad\u307f\u8fbc\u3080<\/h3>\n<p>\u30af\u30ec\u30c7\u30f3\u30b7\u30e3\u30eb\u30d5\u30a1\u30a4\u30eb\uff08\u4f8b\u3048\u3070GCP\u306e\u30b5\u30fc\u30d3\u30b9\u30a2\u30ab\u30a6\u30f3\u30c8\u306eJSON\u30ad\u30fc\u306a\u3069\uff09\u3092Docker\u30a4\u30e1\u30fc\u30b8\u306b\u542b\u3081\u308b\u306e\u306f\u907f\u3051\u305f\u3044\u306e\u3067\u3001DockerOperator\u306e<code>volumes<\/code>\u30d1\u30e9\u30e1\u30fc\u30bf\u3067\u30af\u30ec\u30c7\u30f3\u30b7\u30e3\u30eb\u30d5\u30a1\u30a4\u30eb\u3092\u30de\u30a6\u30f3\u30c8\u3057\u3066DockerOperator\u5185\u304b\u3089\u53c2\u7167\u3067\u304d\u308b\u3088\u3046\u306b\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-python\">DockerOperator(\n    volumes=[&quot;\/Users\/rinoguchi\/workspace\/docker-airflow\/credentials\/:\/credentials\/&quot;],\n)<\/code><\/pre>\n<p>\u5b9f\u306f\u3053\u306e\u8a2d\u5b9a\u3067\u3081\u3063\u3061\u3083\u30cf\u30de\u308a\u307e\u3057\u305f\u3002<br \/>\n<code>volumes=&lt;host_path&gt;:&lt;container_path&gt;<\/code>\u3068\u3044\u3046\u5f62\u5f0f\u3067\u5024\u3092\u8a2d\u5b9a\u3059\u308b\u306e\u3067\u3059\u304c\u3001<code>&lt;host_path&gt;<\/code>\u306e\u90e8\u5206\u306f<b>MacOS\u4e0a\u3067\u306e\u30af\u30ec\u30c7\u30f3\u30b7\u30e3\u30eb\u30d5\u30a1\u30a4\u30eb\u306e\u30d1\u30b9\u3092\u6307\u5b9a\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059<\/b>\u3002\u4eca\u56de\u306e\u69cb\u6210\u3067\u306f\u3001MacOS\u306eDocker\u30a8\u30f3\u30b8\u30f3\u4e0a\u3067Airflow\u306eDocker\u30b3\u30f3\u30c6\u30ca\u3092\u7acb\u3061\u4e0a\u3052\u3001\u305d\u3053\u3067DockerOperator\u3067\u30a2\u30d7\u30ea\u3092\u8d77\u52d5\u3059\u308b\u3082\u306e\u306e\u3001\u30a2\u30d7\u30ea\u306eDocker\u30b3\u30f3\u30c6\u30ca\u81ea\u4f53\u306fMacOS\u4e0a\u306eDocker\u30a8\u30f3\u30b8\u30f3\u3067\u52d5\u304f\u305f\u3081\u3067\u3059\u3002<\/p>\n<p>\u3061\u306a\u307f\u306b\u3001\u9593\u9055\u3063\u3066<code>&#039;valumes=\/usr\/local\/airflow\/credentials\/:\/credentials\/<\/code>\u306e\u3088\u3046\u306bAirflow\u306eDocker\u30b3\u30f3\u30c6\u30ca\u5185\u306e\u30d1\u30b9\u3092\u6307\u5b9a\u3057\u3066\u3044\u305f\u6642\u306f\u3001\u4ee5\u4e0b\u306e\u3088\u3046\u306a\u30a8\u30e9\u30fc\u304c\u51fa\u3066\u307e\u3057\u305f\u3002Docker\u306f\u30de\u30a6\u30f3\u30c8\u5143\u306e\u30d5\u30a1\u30a4\u30eb\u3084\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u304c\u5b58\u5728\u3057\u306a\u3044\u6642\u3001\u7a7a\u306e\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3068\u3057\u3066\u30de\u30a6\u30f3\u30c8\u3057\u3066\u3057\u307e\u3046\u3089\u3057\u3044\u3067\u3059\u3002<\/p>\n<blockquote>\n<p>IsADirectoryError: [Errno 21] Is a directory: '\/credentials\/test-docker-operator-xxxxxxxx.json'<\/p>\n<\/blockquote>\n<h3>DAG\u5b9f\u884c\u6642\u306bJSON\u3067\u30d1\u30e9\u30e1\u30fc\u30bf\u8a2d\u5b9a<\/h3>\n<p>Airflow\u306e\u30d0\u30fc\u30b8\u30e7\u30f3\u304cv1.10.10\u4ee5\u964d\u3067\u3042\u308c\u3070DAG\u5b9f\u884c\u6642\u306b\u3001JSON\u5f62\u5f0f\u3067\u30d1\u30e9\u30e1\u30fc\u30bf\u3092\u8a2d\u5b9a\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u516c\u5f0f\u306e\u30c9\u30ad\u30e5\u30e1\u30f3\u30c8\u306f<a href=\"https:\/\/airflow.apache.org\/docs\/stable\/dag-run.html#passing-parameters-when-triggering-dags\">\u3053\u3061\u3089<\/a>\u3067\u3059\u3002<\/p>\n<p>\u4eca\u56de\u306f\u3001\u300cpython\u306eDocker\u30a4\u30e1\u30fc\u30b8\u306e\u30d0\u30fc\u30b8\u30e7\u30f3\u300d\u3068\u300cpython\u30b3\u30de\u30f3\u30c9\u3078\u306e\u5f15\u6570\u300d\u3092\u30d1\u30e9\u30e1\u30fc\u30bf\u3068\u3057\u3066\u6e21\u3059\u3053\u3068\u3092\u8003\u3048\u307e\u3059\u3002<\/p>\n<p>\u307e\u305a\u306f\u3001DAG\u3092\u5b9f\u88c5\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-python\">    DockerOperator.template_fields = tuple(list(DockerOperator.template_fields) + [&quot;image&quot;])  # NOTE: \u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u5bfe\u8c61\u306b`image`\u3092\u8ffd\u52a0\n    DockerOperator(\n        image=&quot;python:{{ dag_run.conf[&#039;version&#039;] }}&quot;,\n        command=&quot;python {{ dag_run.conf[&#039;args&#039;] }}&quot;,\n    )<\/code><\/pre>\n<p>Airflow\u306f<a href=\"https:\/\/airflow.apache.org\/docs\/stable\/tutorial.html#templating-with-jinja\">\u3053\u3061\u3089<\/a>\u306b\u8a18\u8f09\u3055\u308c\u3066\u3044\u308b\u3088\u3046\u306b\u3001JINJA2\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u3092\u4f7f\u3063\u3066\u30d1\u30e9\u30e1\u30fc\u30bf\u57cb\u3081\u8fbc\u307f\u3068\u30de\u30af\u30ed\u3092\u63d0\u4f9b\u3057\u3066\u3044\u307e\u3059\u3002JINJA2\u3067\u5229\u7528\u53ef\u80fd\u306a\u5909\u6570\u3084\u30de\u30af\u30ed\u306f<a href=\"https:\/\/airflow.apache.org\/docs\/stable\/macros-ref.html\">\u3053\u3061\u3089<\/a>\u3067\u78ba\u8a8d\u3067\u304d\u307e\u3059\u3002<\/p>\n<p>DAG\u5b9f\u884c\u6642\u306e\u30d1\u30e9\u30e1\u30fc\u30bf\u306f\u3001Operator\u306e\u5f15\u6570\u306e\u3046\u3061\u3001JINJA2\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u5bfe\u5fdc\u3057\u305f\u5f15\u6570\u306e\u4e2d\u3067\u306e\u307f\u5229\u7528\u53ef\u80fd\u3067\u3001<code>dag_run.conf<\/code>\u5909\u6570\u306b\u683c\u7d0d\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u5bfe\u5fdc\u3057\u3066\u3044\u308b\u304b\u3069\u3046\u304b\u306f\u3001<a href=\"https:\/\/airflow.apache.org\/docs\/stable\/_api\/airflow\/operators\/docker_operator\/index.html\">API\u30c9\u30ad\u30e5\u30e1\u30f3\u30c8<\/a>\u306b<code>(templated)<\/code>\u3068\u8a18\u8f09\u3055\u308c\u3066\u3044\u308b\u306e\u3067\u305d\u3053\u3067\u78ba\u8a8d\u3059\u308b\u304b\u3001docstring\u3067\u3082\u78ba\u8a8d\u3067\u304d\u307e\u3059\u3002<\/p>\n<p>\u4eca\u56de\u306f\u3001<code>command<\/code>\u306f\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u5bfe\u8c61\u3060\u3063\u305f\u306e\u3067\u3059\u304c\u3001<code>image<\/code>\u306f\u5bfe\u8c61\u5916\u3067\u3057\u305f\u3002\u5b9f\u88c5\u3092\u78ba\u8a8d\u3057\u305f\u3068\u3053\u308d\u3001\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u5bfe\u8c61\u304b\u3069\u3046\u304b\u306fOperator\u30af\u30e9\u30b9\u306e<code>template_fields<\/code>\u3068\u3044\u3046\u30af\u30e9\u30b9\u5909\u6570\u3067\u5b9a\u7fa9\u3055\u308c\u3066\u3044\u305f\u305f\u3081\u3001\u305d\u308c\u3092\u4e0a\u66f8\u304d\u3057\u3066\u3042\u3052\u308b\u3053\u3068\u3067<code>image<\/code>\u3082\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u5bfe\u8c61\u306b\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3057\u305f\u3002<\/p>\n<p>\u3042\u3068\u306f\u3001<code>Configuration JSON (Optional)<\/code>\u306b\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u8a2d\u5b9a\u3057\u3001DAG\u3092\u5b9f\u884c\u3057\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-json\">{\n    &quot;version&quot;: &quot;3.10.0a2&quot;,\n    &quot;args&quot;: &quot;-c \\&quot;from datetime import datetime; print(datetime.now())\\&quot;&quot;\n}<\/code><\/pre>\n<p>\u3061\u3083\u3093\u3068\u671f\u5f85\u901a\u308a\u306b\u5b9f\u884c\u3055\u308c\u3066\u3044\u308b\u3053\u3068\u3092\u30ed\u30b0\u3067\u78ba\u8a8d\u3067\u304d\u307e\u3057\u305f\u3002<\/p>\n<pre><code class=\"language-sh\">[2020-11-07 02:15:28,890] {{docker_operator.py:210}} INFO - Starting docker container from image python:3.10.0a2  # &lt;= \u6307\u5b9a\u3057\u305f\u30d0\u30fc\u30b8\u30e7\u30f3\u306b\u306a\u3063\u3066\u308b\n[2020-11-07 02:15:29,372] {{docker_operator.py:243}} INFO - 2020-11-07 02:15:29.372122  # &lt;= \u6307\u5b9a\u3057\u305f`datetime.now()`\u304c\u5b9f\u884c\u3055\u308c\u3066\u3044\u308b<\/code><\/pre>\n<h3>GCR\uff08Google Cloud Registry\uff09\u4e0a\u306e\u30a4\u30e1\u30fc\u30b8\u3092\u5229\u7528\u3059\u308b<\/h3>\n<p>GCR\u306fPrivate Registory\u306a\u306e\u3067\u8a8d\u8a3c\u3057\u306a\u3044\u3068Docker\u30a4\u30e1\u30fc\u30b8\u3092pull\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u305b\u3093\u3002\u901a\u5e38\u3001GCR\u304b\u3089\u30a4\u30e1\u30fc\u30b8\u3092pull\u3059\u308b\u969b\u306f<code>docker<\/code>\u30b3\u30de\u30f3\u30c9\u306e\u8a2d\u5b9a\u3092\u8ffd\u52a0\u3057\u3066\u3001DCR\u304b\u3089<code>docker pull<\/code>\u969b\u306b\u306f<code>gcloud<\/code>\u30b3\u30de\u30f3\u30c9\u3092\u30af\u30ec\u30c7\u30f3\u30b7\u30e3\u30eb\u30d8\u30eb\u30d1\u30fc\u3068\u3057\u3066\u5229\u7528\u3057\u8a8d\u8a3c\u3092\u884c\u3063\u3066pull\u3057\u307e\u3059\u3002<\/p>\n<p>\u3057\u304b\u3057DockerOperator\u306e\u5834\u5408\u306f<code>gcloud<\/code>\u30b3\u30de\u30f3\u30c9\u3092\u5229\u7528\u3067\u304d\u307e\u305b\u3093\u306e\u3067\u3001<a href=\"https:\/\/airflow.apache.org\/docs\/stable\/_modules\/airflow\/hooks\/docker_hook.html\">DockerHook<\/a>\u3092\u4f7f\u3063\u3066\u8a8d\u8a3c\u3092\u884c\u3044\u307e\u3059\u3002<\/p>\n<p>\u307e\u305a\u306f\u3001WebUI\u306e\u30e1\u30cb\u30e5\u30fc\u304b\u3089\u300cAdmin\u300d&gt;\u300cConnections\u300d\u3092\u30af\u30ea\u30c3\u30af\u3057\u3066Connections\u753b\u9762\u3092\u958b\u304d\u3001\u3055\u3089\u306b\u300cCreate\u300d\u3092\u30af\u30ea\u30c3\u30af\u3057\u3066\u4ee5\u4e0b\u306e\u3088\u3046\u306bConnection\u5b9a\u7fa9\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002<\/p>\n<ul>\n<li>Conn Id: \u597d\u304d\u306a\u5024\u3092<\/li>\n<li>Conn Type: <code>Docker Registry<\/code>\u3092\u9078\u629e<\/li>\n<li>Registry URL: GCR\u306e\u30ea\u30dd\u30b8\u30c8\u30ea\u540d\u3092\u6b63\u78ba\u306b\u8a18\u8f09<\/li>\n<li>Username: <code>_json_key<\/code>\u3092\u56fa\u5b9a\u3067\u8a18\u8f09<\/li>\n<li>Password: \u30b5\u30fc\u30d3\u30b9\u30a2\u30ab\u30a6\u30f3\u30c8\u306eJSON\u30ad\u30fc\u30d5\u30a1\u30a4\u30eb\u306e\u4e2d\u8eab\u306ejson\u5168\u4f53\u3092Copy&amp;Paste<br \/>\n<img src=\"http:\/\/localhost:8000\/wp-content\/uploads\/2021\/01\/20201107152722-600x458.png\" alt=\"\" \/><\/li>\n<\/ul>\n<p>\u6b21\u306b\u3001DockerOperator\u306e\u5f15\u6570\u3092\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u3057\u307e\u3059\u3002<\/p>\n<ul>\n<li>image: GCR\u306e\u30ea\u30dd\u30b8\u30c8\u30ea\u540d\u3092\u8a18\u8f09<\/li>\n<li>docker_conn_id: \u5148\u307b\u3069\u4f5c\u6210\u3057\u3066\u30b3\u30cd\u30af\u30b7\u30e7\u30f3\u5b9a\u7fa9\u306e<code>Conn Id<\/code>\u3092\u8a18\u8f09\n<pre><code class=\"language-python\">DockerOperator(\n    image=&quot;asia.gcr.io\/cloud-registry-test\/python-sample:latest&quot;,\n    docker_conn_id=&quot;gcr_conn&quot;,\n)<\/code><\/pre>\n<\/li>\n<\/ul>\n<p>DAG\u3092\u5b9f\u884c\u3059\u308b\u3068\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u30ed\u30b0\u304c\u51fa\u529b\u3055\u308c\u3001GCR\u304b\u3089<code>python-sample:latest<\/code>\u306e\u30a4\u30e1\u30fc\u30b8\u3092pull\u3057\u3066\u5b9f\u884c\u3067\u304d\u3066\u3044\u308b\u3053\u3068\u304c\u308f\u304b\u308a\u307e\u3057\u305f\u3002<\/p>\n<pre><code class=\"language-sh\">[2020-11-07 07:01:57,489] {{taskinstance.py:901}} INFO - Executing &lt;Task(DockerOperator): pull_from_gcr&gt; on 2020-11-07T07:01:52.591311+00:00\n[2020-11-07 07:01:57,493] {{standard_task_runner.py:54}} INFO - Started process 65 to run task\n[2020-11-07 07:01:57,523] {{standard_task_runner.py:77}} INFO - Running: [&#039;airflow&#039;, &#039;run&#039;, &#039;pull_from_gcr&#039;, &#039;pull_from_gcr&#039;, &#039;2020-11-07T07:01:52.591311+00:00&#039;, &#039;--job_id&#039;, &#039;42&#039;, &#039;--pool&#039;, &#039;default_pool&#039;, &#039;--raw&#039;, &#039;-sd&#039;, &#039;DAGS_FOLDER\/pull_from_gcr.py&#039;, &#039;--cfg_path&#039;, &#039;\/tmp\/tmpwayn7gp0&#039;]\n[2020-11-07 07:01:57,525] {{standard_task_runner.py:78}} INFO - Job 42: Subtask pull_from_gcr\n[2020-11-07 07:01:57,582] {{logging_mixin.py:112}} INFO - Running %s on host %s &lt;TaskInstance: pull_from_gcr.pull_from_gcr 2020-11-07T07:01:52.591311+00:00 [running]&gt; b87faasdfjhe\n[2020-11-07 07:01:57,631] {{base_hook.py:89}} INFO - Using connection to: id: gcr_conn. Host: asia.gcr.io\/cloud-registry-test, Port: None, Schema: None, Login: _json_key, Password: XXXXXXXX, extra: None # &lt;= GCR\u306b\u63a5\u7d9a\u3057\u3066\u3044\u308b\n[2020-11-07 07:01:58,735] {{docker_operator.py:268}} INFO - Pulling docker image asia.gcr.io\/cloud-registry-test\/python-sample:latest # &lt;= GCR\u304b\u3089\u30a4\u30e1\u30fc\u30b8\u3092pull\u3057\u3066\u3044\u308b\n[2020-11-07 07:01:59,825] {{docker_operator.py:273}} INFO - Pulling from cloud-registry-test\/python-sample\n[2020-11-07 07:01:59,829] {{docker_operator.py:273}} INFO - Digest: sha256:7vfq87vqreghjiqwef978vchdqwuirweokj2fed8becd235\n[2020-11-07 07:01:59,839] {{docker_operator.py:273}} INFO - Status: Downloaded newer image for asia.gcr.io\/cloud-registry-test\/python-sample:latest # &lt;= \u30a4\u30e1\u30fc\u30b8\u306e\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9\u5b8c\u4e86\n[2020-11-07 07:01:59,840] {{docker_operator.py:210}} INFO - Starting docker container from image asia.gcr.io\/cloud-registry-test\/python-sample:latest\n[2020-11-07 07:02:00,317] {{docker_operator.py:243}} INFO - python executed. # &lt;= \u30b3\u30f3\u30c6\u30ca\u8d77\u52d5\u6642\u306e`ENTRYPOINT [ &quot;python&quot;, &quot;-c&quot;, &quot;print(&#039;python executed.&#039;)&quot; ]`\u306e\u51fa\u529b\u7d50\u679c\n[2020-11-07 07:02:00,553] {{taskinstance.py:1070}} INFO - Marking task as SUCCESS.dag_id=pull_from_gcr, task_id=pull_from_gcr, execution_date=20201107T070152, start_date=20201107T070157, end_date=20201107T070200\n[2020-11-07 07:02:02,450] {{local_task_job.py:102}} INFO - Task exited with return code 0<\/code><\/pre>\n<h3>Airflow\u518d\u8d77\u52d5\u6642\u306e\u6ce8\u610f\u70b9<\/h3>\n<p>\u4eca\u56de\u306e\u69cb\u6210\u3067\u306f\u3001docker-compose\u3067Airflow\u306eDocker\u30b3\u30f3\u30c6\u30ca\u7fa4\u3092\u8d77\u52d5\u3057\u3066\u3044\u307e\u3059\u3002Dockerfilel\u3092\u66f4\u65b0\u3057\u305f\u969b\u306b\u306f\u30b3\u30f3\u30c6\u30ca\u3092\u518d\u4f5c\u6210\u3057\u305f\u3044\u3067\u3059\u304c\u3001<code>postgres<\/code>\u306bDAG\u306e\u5b9f\u884c\u5c65\u6b74\u3084\u7ba1\u7406\u753b\u9762\u3067\u306e\u8a2d\u5b9a\u304c\u8a18\u9332\u3055\u308c\u3066\u3044\u307e\u3059\u306e\u3067\u3001\u3053\u308c\u3092\u7d76\u5bfe\u306b\u524a\u9664\u3057\u3066\u306f\u3044\u3051\u307e\u305b\u3093\u3002<\/p>\n<p>\u4f55\u3082\u8003\u3048\u305a\u4ee5\u4e0b\u3092\u5b9f\u884c\u3059\u308b\u3068\u3001\u5168\u3066\u306e\u30b3\u30f3\u30c6\u30ca\u304c\u524a\u9664\u3055\u308c\u308b\u306e\u3067\u8981\u6ce8\u610f\u3067\u3059\u3002\u7d76\u5bfe\u306b<code>docker-compose down<\/code>\u3092\u5b9f\u884c\u3057\u306a\u3044\u3088\u3046\u306b\u3057\u307e\u3057\u3087\u3046\u3002<\/p>\n<pre><code class=\"language-sh\">docker-compose -f docker-compose-CeleryExecutor.yml down # &lt;= \u3053\u308c\u3067\u5168\u30b3\u30f3\u30c6\u30ca\u304c\u524a\u9664\u3055\u308c\u308b\uff01\ndocker-compose -f docker-compose-CeleryExecutor.yml up -d<\/code><\/pre>\n<p>\u305f\u3076\u3093\u3001<code>docker-compose down<\/code>\u3055\u3048\u3084\u3089\u306a\u3051\u308c\u3070\u554f\u984c\u306a\u3044\u306e\u3067\u3059\u304c\u3001\u500b\u4eba\u7684\u306b\u306f\u518d\u4f5c\u6210\u304c\u5fc5\u8981\u306a\u6642\u306f\u3001\u4ee5\u4e0b\u306e\u3088\u3046\u306b<code>postgres<\/code>\u3068<code>redis<\/code>\u4ee5\u5916\u306e\u30b3\u30f3\u30c6\u30ca\u3092\u660e\u793a\u7684\u306b\u6307\u5b9a\u3057\u3066\u518d\u4f5c\u6210\u3059\u308b\u3088\u3046\u306b\u3057\u3066\u307e\u3059\u3002\u5909\u66f4\u304c\u306a\u3044\u30b3\u30f3\u30c6\u30ca\u306f\u518d\u4f5c\u6210\u3055\u308c\u307e\u305b\u3093\u3002<code>--no-deps<\/code>\u3092\u3064\u3051\u308b\u3053\u3068\u3067\u4ed6\u306e\u30b3\u30f3\u30c6\u30ca\u3068\u306e\u4f9d\u5b58\u95a2\u4fc2\u3092\u7121\u8996\u3057\u3066\u304f\u308c\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-sh\">docker-compose -f docker-compose-CeleryExecutor.yml up -d --no-deps scheduler flower webserver worker\n> docker-airflow_flower_1  is up-to-date\n> Recreating docker-airflow_webserver_1 ... \n> Recreating docker-airflow_webserver_1 ... done\n> docker-airflow_scheduler_1 is up-to-date\n> docker-airflow_worker_1 is up-to-date<\/code><\/pre>\n<p>\u3061\u306a\u307f\u306b\u3001\u30b3\u30f3\u30c6\u30ca\u3092\u30ea\u30b9\u30bf\u30fc\u30c8\u3057\u305f\u3044\u3060\u3051\u306e\u6642\u306f\u3001\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u3057\u3066\u307e\u3059\u3002<\/p>\n<pre><code class=\"language-sh\">docker-compose -f docker-compose-CeleryExecutor.yml restart webserver<\/code><\/pre>\n<h2>\u3055\u3044\u3054\u306b<\/h2>\n<p>Airflow\u3067DockerOperator\u3092\u52d5\u304b\u3059\u4ef6\u3001\u30cd\u30c3\u30c8\u4e0a\u306b\u60c5\u5831\u304c\u5c11\u306a\u304f\u3066\u601d\u3063\u305f\u3088\u308a\u6642\u9593\u304c\u304b\u304b\u308a\u307e\u3057\u305f\u304c\u3001\u5927\u4f53\u306e\u30b1\u30fc\u30b9\u306f\u7db2\u7f85\u3067\u304d\u305f\u3093\u3058\u3083\u306a\u3044\u304b\u3068\u601d\u3044\u307e\u3059\u306e\u3067\u3001\u3053\u3053\u3089\u3067\u691c\u8a3c\u306f\u7d42\u308f\u308a\u306b\u3057\u3088\u3046\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n<p>\u3061\u306a\u307f\u306b\u3001docker-airflow\u3092fork\u3057\u3066\u691c\u8a3c\u3057\u305f\u30ea\u30dd\u30b8\u30c8\u30ea\u306f <a href=\"https:\/\/github.com\/rinoguchi\/docker-airflow\">\u3053\u3061\u3089<\/a> \u306b\u306a\u308a\u307e\u3059\u306e\u3067\u3001\u8208\u5473\u304c\u3042\u308c\u3070\u53c2\u7167\u304f\u3060\u3055\u3044\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>Airflow\u306f\u3001\u30ef\u30fc\u30af\u30d5\u30ed\u30fc\u3092\u30d7\u30ed\u30b0\u30e9\u30e0\u3067\u4f5c\u6210\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u3001\u304a\u3088\u3073\u76e3\u8996\u3059\u308b\u305f\u3081\u306e\u30d7\u30e9\u30c3\u30c8\u30d5\u30a9\u30fc\u30e0\u3067\u3059\u3002\u4ed5\u4e8b\u3067Airflow\u3092\u63a1\u7528\u3059\u308b\u3053\u3068\u306b\u3057\u305f\u306e\u3067\u3001\u4eca\u56de\u306fAirflow\u3092\u30ed\u30fc\u30ab\u30ebPC\u4e0a\u3067\u52d5\u304b\u3057\u3066\u3001DockerOperator\u3067\u4efb\u610f\u306eDocker\u30b3\u30f3\u30c6\u30ca\u3092\u5b9f\u884c\u3057\u3064\u3064\u3001\u5fc5\u8981\u306a\u8981\u7d20\uff08DAG\u5b9f\u884c\u6642\u306e\u30d1\u30e9\u30e1\u30fc\u30bf\u6307\u5b9a\u3068\u304b\u74b0\u5883\u5909\u6570\u6e21\u3057\u306a\u3069\uff09\u3092\u8a66\u3057\u3066\u3044\u304d\u305f\u3044\u3068\u601d\u3044\u307e\u3059\u3002 Airflow\u306b\u3064\u3044\u3066 \u516c\u5f0f\u306e\u30c8\u30c3\u30d7\u30da\u30fc\u30b8\u304b\u3089\u5f15\u3063\u5f35\u3063\u3066\u304d\u305f\u3060\u3051\u3067\u3059\u304c\u3001\u4ee5\u4e0b\u306e\u3088\u3046\u306a\u539f\u5247\u30fb\u7279\u5fb4\u304c\u3042\u308b\u305d\u3046\u3067\u3059\u3002 \u539f\u5247 \u30b9\u30b1\u30fc\u30e9\u30d6\u30eb Airflow\u306f\u30e2\u30b8\u30e5\u30fc\u30eb\u5316\u3055\u308c\u305f\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u3092\u5099\u3048\u3066\u3044\u3066\u3001Message\u30ad\u30e5\u30fc\u3092\u4f7f\u7528\u3057\u3066\u4efb\u610f\u306e\u6570\u306e\u30ef\u30fc\u30ab\u30fc\u3092\u8abf\u6574\u3067\u304d\u3001\u7121\u9650\u306b\u62e1\u5f35\u3067\u304d\u308b \u52d5\u7684 Airflow\u306e\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u306fPython\u3067\u5b9a\u7fa9\u3055\u308c\u3066\u304a\u308a\u3001\u52d5\u7684\u306b\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u3092\u751f\u6210\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b \u62e1\u5f35\u53ef\u80fd \u72ec\u81ea\u306e\u6f14\u7b97\u5b50\u3092\u7c21\u5358\u306b\u5b9a\u7fa9\u3057\u3001\u30e9\u30a4\u30d6\u30e9\u30ea\u3092\u62e1\u5f35\u3057\u3066 <a href=\"http:\/\/localhost:8000\/2020\/11\/airflow-dockeroperator.html\" class=\"read-more button-fancy -red\"><span class=\"btn-arrow\"><\/span><span class=\"twp-read-more text\">Continue Reading<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[18,13],"tags":[],"_links":{"self":[{"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/posts\/167"}],"collection":[{"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/comments?post=167"}],"version-history":[{"count":0,"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/posts\/167\/revisions"}],"wp:attachment":[{"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/media?parent=167"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/categories?post=167"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/localhost:8000\/wp-json\/wp\/v2\/tags?post=167"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}