diff --git a/restscrape/celery.py b/restscrape/celery.py index 2410e9b..b4a0f6b 100644 --- a/restscrape/celery.py +++ b/restscrape/celery.py @@ -1,12 +1,10 @@ +from django.conf import settings from celery import Celery -from restscrape.utils import get_tab app = Celery("restscrape_funcs") -app.conf.broker_url = "redis://localhost:6379/0" -app.conf.result_backend = "redis://localhost:6379/0" - - -@app.task -def fetch_page(url, wait_for=0, proxy=None, use_adblock=True): - with get_tab(proxy=proxy, use_adblock=use_adblock) as tab: - return tab.open(url, wait_for=wait_for) +app.conf.broker_url = getattr( + settings, "MESSAGE_BROKER_URL", "amqp://myuser:mypassword@localhost:5672/myvhost" +) +app.conf.result_backend = getattr( + settings, "RESULT_BACKEND_URL", "redis://localhost:6379/0" +) diff --git a/restscrape/serializers.py b/restscrape/serializers.py index 236cd38..07f11b1 100644 --- a/restscrape/serializers.py +++ b/restscrape/serializers.py @@ -1 +1,15 @@ from rest_framework import serializers + +from restscrape.models import ScrapeRequest, PendingScrapingResponse + + +class ScrapeRequestSerializer(serializers.ModelSerializer): + class Meta: + model = ScrapeRequest + fields = ("blocking", "url", "use_adblock", "xpath_labels", "wait_for", "proxy") + + +class PendingScrapingResponseSerializer(serializers.ModelSerializer): + class Meta: + model = PendingScrapingResponse + fields = "id" diff --git a/restscrape/urls.py b/restscrape/urls.py new file mode 100644 index 0000000..1cfbebb --- /dev/null +++ b/restscrape/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + + +from restscrape.views import SubmitScrapeRequestView + + +urlpatterns = [path("submit/", SubmitScrapeRequestView.as_view(), name="scrape_submit")] diff --git a/restscrape/views.py b/restscrape/views.py index 91ea44a..e1202d0 100644 --- a/restscrape/views.py +++ b/restscrape/views.py @@ -1,3 +1,18 @@ -from django.shortcuts import render +from rest_framework.views import APIView +from rest_framework.generics import CreateAPIView +from rest_framework.response import Response -# Create your views here. +from restscrape.serializers import ( + ScrapeRequestSerializer, + PendingScrapingResponseSerializer, +) + + +class SubmitScrapeRequestView(APIView): + def post(self, request, format=None): + serializer = ScrapeRequestSerializer(data=request.data) + if serializer.is_valid(): + submission = serializer.create(serializer.validated_data) + pending = submission.submit() + pending_serializer = PendingScrapingResponseSerializer(pending) + return Response(data=pending_serializer.data)