PostScrape/src/post_scrape/pipelines.py

import sqlite3
import psycopg2


class PostgresqlPostPipeline:
    def __init__(self):
        self.id = 1
        DB_NAME = "cars"
        DB_USER = "ayxan"
        DB_PASS = "admin"
        DB_HOST = "127.0.0.1"
        DB_PORT = "5432"

        self.con = psycopg2.connect(database=DB_NAME,
                                user=DB_USER,
                                password=DB_PASS,
                                host=DB_HOST,
                                port=DB_PORT)
        self.cur = self.con.cursor()

        self.cur.execute("""
        CREATE TABLE IF NOT EXISTS cars(
            id INTEGER PRIMARY KEY,
            url TEXT,
            avto_salon TEXT,
            description TEXT,
            city TEXT,
            brand TEXT,
            model TEXT,
            year INTEGER,
            category TEXT,
            color TEXT,
            engine_volume INTEGER,
            engine_power INTEGER,
            fuel_type TEXT,
            mileage INTEGER,
            mileage_type TEXT,
            transmission TEXT,
            gear TEXT,
            price INTEGER,
            currency TEXT,
            loan TEXT,
            barter TEXT,
            market TEXT,
            seats_count TEXT,
            prior_owners_count TEXT,
            crashed TEXT,
            painted TEXT
        )
        """)
        # IMAGE
        self.cur.execute("""
        CREATE TABLE IF NOT EXISTS images(
            post_id INTEGER,
            url TEXT,
            FOREIGN KEY(post_id) REFERENCES cars(id)
        )
        """)
        # PHONE
        self.cur.execute("""
        CREATE TABLE IF NOT EXISTS phones(
            post_id INTEGER,
            phone TEXT,
            FOREIGN KEY(post_id) REFERENCES cars(id)
        )
        """)
        # EXTRA_FIELDS
        self.cur.execute("""
        CREATE TABLE IF NOT EXISTS extra_fields(
            post_id INTEGER,
            extra_field TEXT,
            FOREIGN KEY(post_id) REFERENCES cars(id)
        )
        """)
        
        self.cur.execute("SELECT * FROM cars ORDER BY id DESC LIMIT 1")
        result = self.cur.fetchone()
        if result is not None: self.id = result[0] + 1

    def process_item(self, item, spider):
        self.cur.execute("""
            INSERT INTO cars (id, url, avto_salon, description, city, brand, model, year, category, color, engine_volume, engine_power, fuel_type, mileage, mileage_type, transmission, gear, price, currency, loan, barter, market, seats_count, prior_owners_count, crashed, painted) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
        """,
        (
            self.id,
            item['url'],
            item['avto_salon'],
            ' '.join(item['description']),
            item['city'],
            item['brand'],
            item['model'],
            item['year'],
            item['category'],
            item['color'],
            item['engine_volume'],
            item['engine_power'],
            item['fuel_type'],
            item['mileage'],
            item['mileage_type'],
            item['transmission'],
            item['gear'],
            item['price'],
            item['currency'],
            item['loan'],
            item['barter'],
            item['market'],
            item['seats_count'],
            item['prior_owners_count'],
            item['crashed'],
            item['painted']
        ))
        # IMAGE
        for image in item['images']:
            self.cur.execute("""INSERT INTO images (post_id, url) VALUES (%s, %s)""",
                            (self.id, image))
        # PHONE
        for phone in item['phone']:
            self.cur.execute("""INSERT INTO phones (post_id, phone) VALUES (%s, %s)""",
                            (self.id, phone))
        # EXTRA_FIELDS
        for field in item['extra_fields']:
            self.cur.execute("""INSERT INTO extra_fields (post_id, extra_field) VALUES (%s, %s)""",
                            (self.id, field))

        self.con.commit()
        self.id += 1
        return item

class SqlitePostPipeline:
    def __init__(self):
        self.id = 1
        self.con = sqlite3.connect('cars.db')
        self.cur = self.con.cursor()

        self.cur.execute("""
        CREATE TABLE IF NOT EXISTS cars(
            id INTEGER PRIMARY KEY,
            url TEXT,
            avto_salon TEXT,
            description TEXT,
            city TEXT,
            brand TEXT,
            model TEXT,
            year INTEGER,
            category TEXT,
            color TEXT,
            engine_volume INTEGER,
            engine_power INTEGER,
            fuel_type TEXT,
            mileage INTEGER,
            mileage_type TEXT,
            transmission TEXT,
            gear TEXT,
            price INTEGER,
            currency TEXT,
            loan TEXT,
            barter TEXT,
            market TEXT,
            seats_count TEXT,
            prior_owners_count TEXT,
            crashed TEXT,
            painted TEXT
        )
        """)
        # IMAGE
        self.cur.execute("""
        CREATE TABLE IF NOT EXISTS images(
            post_id INTEGER,
            url TEXT,
            FOREIGN KEY(post_id) REFERENCES cars(id)
        )
        """)
        # PHONE
        self.cur.execute("""
        CREATE TABLE IF NOT EXISTS phones(
            post_id INTEGER,
            phone TEXT,
            FOREIGN KEY(post_id) REFERENCES cars(id)
        )
        """)
        # EXTRA_FIELDS
        self.cur.execute("""
        CREATE TABLE IF NOT EXISTS extra_fields(
            post_id INTEGER,
            extra_field TEXT,
            FOREIGN KEY(post_id) REFERENCES cars(id)
        )
        """)
        
        self.cur.execute("SELECT * FROM cars ORDER BY id DESC LIMIT 1")
        result = self.cur.fetchone()
        if result is not None: self.id = result[0] + 1

    def process_item(self, item, spider):
        self.cur.execute("""
            INSERT INTO cars (id, url, avto_salon, description, city, brand, model, year, category, color, engine_volume, engine_power, fuel_type, mileage, mileage_type, transmission, gear, price, currency, loan, barter, market, seats_count, prior_owners_count, crashed, painted) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
        (
            self.id,
            item['url'],
            item['avto_salon'],
            ' '.join(item['description']),
            item['city'],
            item['brand'],
            item['model'],
            item['year'],
            item['category'],
            item['color'],
            item['engine_volume'],
            item['engine_power'],
            item['fuel_type'],
            item['mileage'],
            item['mileage_type'],
            item['transmission'],
            item['gear'],
            item['price'],
            item['currency'],
            item['loan'],
            item['barter'],
            item['market'],
            item['seats_count'],
            item['prior_owners_count'],
            item['crashed'],
            item['painted']
        ))
        # IMAGE
        for image in item['images']:
            self.cur.execute("""INSERT INTO images (post_id, url) VALUES (?, ?)""",
                            (self.id, image))
        # PHONE
        for phone in item['phone']:
            self.cur.execute("""INSERT INTO phones (post_id, phone) VALUES (?, ?)""",
                            (self.id, phone))
        # EXTRA_FIELDS
        for field in item['extra_fields']:
            self.cur.execute("""INSERT INTO extra_fields (post_id, extra_field) VALUES (?, ?)""",
                            (self.id, field))

        self.con.commit()
        self.id += 1
        return item