more

Author: waleedlatif1

apps/sim/app/(landing)/components/structured-data.tsx

@@ -42,9 +42,6 @@ export default function StructuredData() {
         publisher: {
           '@id': 'https://sim.ai/#organization',
         },
-        // Note: SearchAction removed - no public /search page exists
-        // Add back when search functionality is implemented:
-        // potentialAction: [{ '@type': 'SearchAction', target: { urlTemplate: '...' } }]
         inLanguage: 'en-US',
       },
       {

apps/sim/app/robots.ts

@@ -4,72 +4,120 @@ import { getBaseUrl } from '@/lib/core/utils/urls'
 export default function robots(): MetadataRoute.Robots {
   const baseUrl = getBaseUrl()
 
+  const disallowedPaths = [
+    '/api/',
+    '/workspace/',
+    '/chat/',
+    '/playground/',
+    '/resume/',
+    '/invite/',
+    '/unsubscribe/',
+    '/w/',
+    '/_next/',
+    '/private/',
+  ]
+
   return {
     rules: [
       {
         userAgent: '*',
         allow: '/',
-        disallow: ['/api/', '/workspace/', '/_next/', '/private/', '/*.json$'],
+        disallow: disallowedPaths,
       },
       {
         userAgent: 'Googlebot',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
         userAgent: 'Bingbot',
         allow: '/',
+        disallow: disallowedPaths,
+      },
+      {
+        userAgent: 'YandexBot',
+        allow: '/',
+        disallow: disallowedPaths,
+      },
+      {
+        userAgent: 'Baiduspider',
+        allow: '/',
+        disallow: disallowedPaths,
       },
       {
         userAgent: 'GPTBot',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
         userAgent: 'ChatGPT-User',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
         userAgent: 'OAI-SearchBot',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
         userAgent: 'ClaudeBot',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
-        userAgent: 'Claude-Web',
+        userAgent: 'Claude-SearchBot',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
-        userAgent: 'anthropic-ai',
+        userAgent: 'Google-Extended',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
-        userAgent: 'Google-Extended',
+        userAgent: 'PerplexityBot',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
-        userAgent: 'PerplexityBot',
+        userAgent: 'Meta-ExternalAgent',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
-        userAgent: 'Perplexity-User',
+        userAgent: 'FacebookBot',
         allow: '/',
+        disallow: disallowedPaths,
+      },
+      {
+        userAgent: 'Applebot',
+        allow: '/',
+        disallow: disallowedPaths,
       },
       {
         userAgent: 'Applebot-Extended',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
-        userAgent: 'CCBot',
+        userAgent: 'Amazonbot',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
-        userAgent: 'FacebookBot',
+        userAgent: 'Bytespider',
+        allow: '/',
+        disallow: disallowedPaths,
+      },
+      {
+        userAgent: 'CCBot',
         allow: '/',
+        disallow: disallowedPaths,
       },
       {
         userAgent: 'cohere-ai',
         allow: '/',
+        disallow: disallowedPaths,
       },
     ],
     sitemap: `${baseUrl}/sitemap.xml`,