diff --git a/apps/admin-web/index.html b/apps/admin-web/index.html index b65b325..444c643 100644 --- a/apps/admin-web/index.html +++ b/apps/admin-web/index.html @@ -3,6 +3,7 @@ + 杜康好客 · HQ 管理后台 diff --git a/apps/admin-web/public/robots.txt b/apps/admin-web/public/robots.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/apps/admin-web/public/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/apps/h5-partner/index.html b/apps/h5-partner/index.html index c5fba1d..4998d25 100644 --- a/apps/h5-partner/index.html +++ b/apps/h5-partner/index.html @@ -3,6 +3,7 @@ + 城市合伙人 diff --git a/apps/h5-partner/public/robots.txt b/apps/h5-partner/public/robots.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/apps/h5-partner/public/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/apps/h5-shop/index.html b/apps/h5-shop/index.html index efe4b31..446ed46 100644 --- a/apps/h5-shop/index.html +++ b/apps/h5-shop/index.html @@ -3,6 +3,7 @@ + 门店管理中心 diff --git a/apps/h5-shop/public/robots.txt b/apps/h5-shop/public/robots.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/apps/h5-shop/public/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/deploy/nginx-dukang-staging.conf b/deploy/nginx-dukang-staging.conf index 318d536..f69078e 100644 --- a/deploy/nginx-dukang-staging.conf +++ b/deploy/nginx-dukang-staging.conf @@ -104,6 +104,7 @@ server { client_max_body_size 50m; include /opt/dukang-staging/deploy/nginx-mp-verify-staging.conf; + include /opt/dukang-staging/deploy/nginx-no-crawler.conf; location /api/ { proxy_pass http://127.0.0.1:8190; @@ -142,6 +143,7 @@ server { client_max_body_size 50m; include /opt/dukang-staging/deploy/nginx-mp-verify-staging.conf; + include /opt/dukang-staging/deploy/nginx-no-crawler.conf; location /api/ { proxy_pass http://127.0.0.1:8190; @@ -179,6 +181,8 @@ server { client_max_body_size 50m; + include /opt/dukang-staging/deploy/nginx-no-crawler.conf; + location /api/ { proxy_pass http://127.0.0.1:8190; proxy_http_version 1.1; diff --git a/deploy/nginx-dukanghaoke.conf b/deploy/nginx-dukanghaoke.conf index ea7b433..c6d519e 100644 --- a/deploy/nginx-dukanghaoke.conf +++ b/deploy/nginx-dukanghaoke.conf @@ -94,6 +94,7 @@ server { client_max_body_size 50m; include /opt/dukang/deploy/nginx-mp-verify.conf; + include /opt/dukang/deploy/nginx-no-crawler.conf; location /api/ { proxy_pass http://127.0.0.1:8090; @@ -132,6 +133,7 @@ server { client_max_body_size 50m; include /opt/dukang/deploy/nginx-mp-verify.conf; + include /opt/dukang/deploy/nginx-no-crawler.conf; location /api/ { proxy_pass http://127.0.0.1:8090; @@ -169,6 +171,8 @@ server { client_max_body_size 50m; + include /opt/dukang/deploy/nginx-no-crawler.conf; + location /api/ { proxy_pass http://127.0.0.1:8090; proxy_http_version 1.1; diff --git a/deploy/nginx-no-crawler.conf b/deploy/nginx-no-crawler.conf new file mode 100644 index 0000000..7e8ffd4 --- /dev/null +++ b/deploy/nginx-no-crawler.conf @@ -0,0 +1,11 @@ +# 禁止搜索引擎抓取(shop / partner / admin,含测试域) +# 合规爬虫读 robots.txt;X-Robots-Tag 覆盖不先读 robots.txt 的抓取器 + +add_header X-Robots-Tag "noindex, nofollow, noarchive" always; + +location = /robots.txt { + default_type text/plain; + charset utf-8; + add_header X-Robots-Tag "noindex, nofollow, noarchive" always; + return 200 "User-agent: *\nDisallow: /\n"; +}