Index

_

__init__() (scrapy_item_ingest.config.settings.Settings method) (scrapy_item_ingest.DbInsertPipeline method) (scrapy_item_ingest.extensions.base.BaseExtension method) (scrapy_item_ingest.LoggingExtension method) (scrapy_item_ingest.pipelines.base.BasePipeline method) (scrapy_item_ingest.RequestsPipeline method)

B

BaseExtension (class in scrapy_item_ingest.extensions.base)

BasePipeline (class in scrapy_item_ingest.pipelines.base)

C

close_spider() (scrapy_item_ingest.DbInsertPipeline method) (scrapy_item_ingest.pipelines.base.BasePipeline method)

create_tables (scrapy_item_ingest.config.settings.Settings property)

D

db_items_table (scrapy_item_ingest.config.settings.Settings property) db_logs_table (scrapy_item_ingest.config.settings.Settings property) db_requests_table (scrapy_item_ingest.config.settings.Settings property) db_type (scrapy_item_ingest.config.settings.Settings property) db_url (scrapy_item_ingest.config.settings.Settings property)

DbInsertPipeline (class in scrapy_item_ingest) DEFAULT_ITEMS_TABLE (scrapy_item_ingest.config.settings.Settings attribute) DEFAULT_LOGS_TABLE (scrapy_item_ingest.config.settings.Settings attribute) DEFAULT_REQUESTS_TABLE (scrapy_item_ingest.config.settings.Settings attribute) DEFAULT_TIMEZONE (scrapy_item_ingest.config.settings.Settings attribute)

E

engine_stopped() (scrapy_item_ingest.LoggingExtension method)

F

from_crawler() (scrapy_item_ingest.DbInsertPipeline class method) (scrapy_item_ingest.extensions.base.BaseExtension class method) (scrapy_item_ingest.extensions.base.BaseExtension method) (scrapy_item_ingest.LoggingExtension class method) (scrapy_item_ingest.pipelines.base.BasePipeline class method) (scrapy_item_ingest.RequestsPipeline class method)

from_crawler_settings() (in module scrapy_item_ingest.config.settings)

G

get_current_datetime() (in module scrapy_item_ingest.utils.time) get_database_connection() (scrapy_item_ingest.extensions.base.BaseExtension method) get_database_url() (in module scrapy_item_ingest.config.settings) get_identifier_column() (scrapy_item_ingest.config.settings.Settings static method) get_identifier_info() (scrapy_item_ingest.extensions.base.BaseExtension method) (scrapy_item_ingest.pipelines.base.BasePipeline method)

get_identifier_value() (scrapy_item_ingest.config.settings.Settings method) get_job_id() (scrapy_item_ingest.extensions.base.BaseExtension method) get_request_fingerprint() (in module scrapy_item_ingest.utils.fingerprint), [1] get_table_name() (in module scrapy_item_ingest.config.settings) get_tz() (scrapy_item_ingest.config.settings.Settings method)

I

ItemsPipeline (class in scrapy_item_ingest)

L

log_message() (scrapy_item_ingest.extensions.base.BaseExtension method)

log_request() (scrapy_item_ingest.RequestsPipeline method) LoggingExtension (class in scrapy_item_ingest)

M

module scrapy_item_ingest.utils.fingerprint, [1] scrapy_item_ingest.utils.serialization scrapy_item_ingest.utils.time

O

open_spider() (scrapy_item_ingest.DbInsertPipeline method) (scrapy_item_ingest.pipelines.base.BasePipeline method)

P

process_item() (scrapy_item_ingest.DbInsertPipeline method) (scrapy_item_ingest.ItemsPipeline method)

R

request_scheduled() (scrapy_item_ingest.RequestsPipeline method)

RequestsPipeline (class in scrapy_item_ingest) response_received() (scrapy_item_ingest.RequestsPipeline method)

S

scrapy_item_ingest.utils.fingerprint module, [1] scrapy_item_ingest.utils.serialization module scrapy_item_ingest.utils.time module

serialize_item_data() (in module scrapy_item_ingest.utils.serialization) Settings (class in scrapy_item_ingest.config.settings) spider_closed() (scrapy_item_ingest.LoggingExtension method) spider_opened() (scrapy_item_ingest.LoggingExtension method)

V

validate() (in module scrapy_item_ingest.config.settings)

validate_settings() (in module scrapy_item_ingest.config.settings), [1]