import scrapy
from scrapy.crawler import CrawlerProcess
import sys
import os
import django

# Setup django environment
# __file__ is c:\django\chat bot\chatpro\chatapp\sitemap_spider.py
# So project dir is c:\django\chat bot\chatpro
project_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(project_dir)
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'chatpro.settings')
os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
django.setup()

from chatapp.models import Company, Product

class SitemapProductSpider(scrapy.Spider):
    name = "sitemap_product_spider"

    def __init__(self, company_id=None, sitemap_url=None, *args, **kwargs):
        super(SitemapProductSpider, self).__init__(*args, **kwargs)
        self.company_id = company_id
        self.sitemap_url = sitemap_url
        self.start_urls = [sitemap_url] if sitemap_url else []
        self.company = Company.objects.get(id=self.company_id) if company_id else None

    def parse(self, response):
        # Register namespace for sitemap XML
        response.selector.register_namespace('d', 'http://www.sitemaps.org/schemas/sitemap/0.9')
        
        # If it's a sitemap index, yield requests for each sitemap
        sitemaps = response.xpath('//d:sitemap/d:loc/text()').getall()
        for sm in sitemaps:
            yield scrapy.Request(sm, callback=self.parse)
        
        # If it's a urlset, yield requests for each URL
        urls = response.xpath('//d:url/d:loc/text()').getall()
        # Fallback to plain regex if namespace fails
        if not urls:
            urls = response.xpath('//*[local-name()="url"]/*[local-name()="loc"]/text()').getall()
        if not sitemaps and not urls:
            sitemaps = response.xpath('//*[local-name()="sitemap"]/*[local-name()="loc"]/text()').getall()
            for sm in sitemaps:
                yield scrapy.Request(sm, callback=self.parse)
                
        for url in urls:
            yield scrapy.Request(url, callback=self.parse_product)

    def parse_product(self, response):
        if not self.company:
            return

        # Basic extraction using OpenGraph and meta tags
        title = response.xpath('//meta[@property="og:title"]/@content').get() or response.xpath('//title/text()').get()
        price = response.xpath('//meta[@property="product:price:amount"]/@content').get() or response.xpath('//meta[@name="twitter:data1"]/@content').get() or response.xpath('//span[contains(@class, "price")]/text()').get()
        brand = response.xpath('//meta[@property="product:brand"]/@content').get() or response.xpath('//meta[@name="brand"]/@content').get()
        image_url = response.xpath('//meta[@property="og:image"]/@content').get()
        description = response.xpath('//meta[@property="og:description"]/@content').get() or response.xpath('//meta[@name="description"]/@content').get()

        if title:
            title = title.strip()
            # If title is very long, truncate
            if len(title) > 500:
                title = title[:497] + '...'
        if price:
            price = price.strip()
            if len(price) > 100:
                price = price[:97] + '...'
        if brand:
            brand = brand.strip()
            if len(brand) > 255:
                brand = brand[:252] + '...'

        # Try to find JSON-LD
        import json
        json_ld_scripts = response.xpath('//script[@type="application/ld+json"]/text()').getall()
        for script in json_ld_scripts:
            try:
                data = json.loads(script)
                # Some JSON-LD are lists
                if isinstance(data, dict):
                    data = [data]
                for item in data:
                    if item.get('@type') == 'Product':
                        if not title and item.get('name'):
                            title = item.get('name')
                        if not description and item.get('description'):
                            description = item.get('description')
                        if not image_url and item.get('image'):
                            img = item.get('image')
                            if isinstance(img, list):
                                image_url = img[0]
                            elif isinstance(img, str):
                                image_url = img
                            elif isinstance(img, dict) and img.get('url'):
                                image_url = img.get('url')
                        if not brand and item.get('brand') and isinstance(item.get('brand'), dict):
                            brand = item.get('brand').get('name')
                        if not price and item.get('offers') and isinstance(item.get('offers'), dict):
                            price = str(item.get('offers').get('price', ''))
                            currency = item.get('offers').get('priceCurrency', '')
                            if currency and price:
                                price = f"{price} {currency}"
            except Exception:
                pass

        # Save to database
        if title: # Only save if we got at least a title
            Product.objects.update_or_create(
                company=self.company,
                url=response.url,
                defaults={
                    'title': title,
                    'price': price,
                    'brand': brand,
                    'image_url': image_url,
                    'description': description
                }
            )

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("Usage: python sitemap_spider.py <company_id> <sitemap_url>")
        sys.exit(1)
    
    company_id = sys.argv[1]
    sitemap_url = sys.argv[2]
    
    process = CrawlerProcess({
        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
        'LOG_LEVEL': 'INFO'
    })

    process.crawl(SitemapProductSpider, company_id=company_id, sitemap_url=sitemap_url)
    process.start()