chore(deploy): shop/partner/admin 禁止搜索引擎抓取

nginx 直出 robots.txt 并加 X-Robots-Tag，避免内部三端被收录。

Co-authored-by: Cursor <cursoragent@cursor.com>

deploy/nginx-dukang-staging.conf

@@ -104,6 +104,7 @@ server {
     client_max_body_size 50m;
 
     include /opt/dukang-staging/deploy/nginx-mp-verify-staging.conf;
+    include /opt/dukang-staging/deploy/nginx-no-crawler.conf;
 
     location /api/ {
         proxy_pass http://127.0.0.1:8190;
@@ -142,6 +143,7 @@ server {
     client_max_body_size 50m;
 
     include /opt/dukang-staging/deploy/nginx-mp-verify-staging.conf;
+    include /opt/dukang-staging/deploy/nginx-no-crawler.conf;
 
     location /api/ {
         proxy_pass http://127.0.0.1:8190;
@@ -179,6 +181,8 @@ server {
 
     client_max_body_size 50m;
 
+    include /opt/dukang-staging/deploy/nginx-no-crawler.conf;
+
     location /api/ {
         proxy_pass http://127.0.0.1:8190;
         proxy_http_version 1.1;

deploy/nginx-dukanghaoke.conf

@@ -94,6 +94,7 @@ server {
     client_max_body_size 50m;
 
     include /opt/dukang/deploy/nginx-mp-verify.conf;
+    include /opt/dukang/deploy/nginx-no-crawler.conf;
 
     location /api/ {
         proxy_pass http://127.0.0.1:8090;
@@ -132,6 +133,7 @@ server {
     client_max_body_size 50m;
 
     include /opt/dukang/deploy/nginx-mp-verify.conf;
+    include /opt/dukang/deploy/nginx-no-crawler.conf;
 
     location /api/ {
         proxy_pass http://127.0.0.1:8090;
@@ -169,6 +171,8 @@ server {
 
     client_max_body_size 50m;
 
+    include /opt/dukang/deploy/nginx-no-crawler.conf;
+
     location /api/ {
         proxy_pass http://127.0.0.1:8090;
         proxy_http_version 1.1;

deploy/nginx-no-crawler.conf

@@ -0,0 +1,11 @@
+# 禁止搜索引擎抓取（shop / partner / admin，含测试域）
+# 合规爬虫读 robots.txt；X-Robots-Tag 覆盖不先读 robots.txt 的抓取器
+
+add_header X-Robots-Tag "noindex, nofollow, noarchive" always;
+
+location = /robots.txt {
+    default_type text/plain;
+    charset utf-8;
+    add_header X-Robots-Tag "noindex, nofollow, noarchive" always;
+    return 200 "User-agent: *\nDisallow: /\n";
+}