Rework the backend api

api/karumedia/__init__.py

@@ -2,10 +2,12 @@ import json
 from pathlib import Path
 
 import falcon
+import mongoengine
 
 from .tools import JsonRequest, JsonResponse, error_handler
 from .controllers import *
 
+
 class AboutResource():
 
     def on_get(self, req, resp):
@@ -35,15 +37,19 @@ class AboutResource():
 
         resp.body = json.dumps(r, indent=4)
 
+mongoengine.connect('KaruMedia', connect=False)
 
 app = application = falcon.API(request_type=JsonRequest, response_type=JsonResponse)
 app.add_error_handler(ValueError, error_handler)
+app.add_error_handler(mongoengine.DoesNotExist, error_handler)
 app.add_route("/", AboutResource())
 
-path = Path("/home/arti/Videod")
+path = Path("/srv/media/")
 
 app.add_route("/movies", MoviesCollection(path))
 app.add_route("/movies/{movie}", MoviesResource(path))
-app.add_route("/movies/{movie}/stream", MovieStreamUrlResource(path))
 
 app.add_route("/magnet", MagnetResource())
+
+app.add_route("/tasks", TaskCollection())
+app.add_route("/tasks/{task_id}", TaskResource())

api/karumedia/controllers.py

@@ -1,8 +1,14 @@
 import os
 import re
 import json
+from . import tasks
 from falcon import HTTPInternalServerError, HTTP_201
+from bson.objectid import ObjectId
 from .tools import TODOException
+from .models import *
+from pprint import pprint
+from urllib.parse import quote
+
 
 movie_name_and_year = re.compile("(.*)\((.*)\)")
 
@@ -10,55 +16,52 @@ class BaseMovieResource():
     def __init__(self, path):
         self.path = path
 
+def build_movie_object(media):
+    mobj = {
+        "title":media.title,
+        "title_english":media.title,
+        "title_long": f"{media.title} ({media.year})",
+        "year":media.year,
+        "imdb_code": media.imdb_id,
+        "rating": media.rating,
+        "summary": media.plot_outline,
+        "synopsis": media.plot_outline,
+        "mpa_rating": media.mpa_rating,
+        "genres": media.genres,
+        "yt_trailer_code": media.yt_trailer_code,
+        "state": "ok",
+        "id": media.id
+    }
+    try:
+        mobj["runtime"] = media.runtime // 100
+    except:
+        pass
+    if media.plots:
+        mobj["description_full"] = media.plots[0]
+
+    relevant_tasks = list(Task.objects(imdb_id=media.imdb_id, state__ne="done"))
+    if relevant_tasks:
+        pprint(relevant_tasks)
+        mobj["state"] = "tasks_running"
+        mobj["relevant_tasks"] = [task.id for task in relevant_tasks]
+
+    mobj["streams"] = {}
+    stream_urls = list(MediaFile.objects(media=media))
+    if stream_urls:
+        mobj["streams"] = {media_file.resolution: media_file.url for media_file in stream_urls if media_file.mimetype.startswith("video")}
+
+    mobj["small_cover_image"] = f"https://media.arti.ee/Filmid/{quote(mobj['title_long'])}/cover.jpg"
+    mobj["medium_cover_image"] = mobj["small_cover_image"]
+    mobj["large_cover_image"] = mobj["medium_cover_image"]
+
+    return mobj
+
 class MoviesCollection(BaseMovieResource):
 
     def on_get(self, req, resp):
-        movie_paths = [p for p in (self.path / 'Filmid').iterdir() if p.is_dir()]
         movies = []
-        for movie_path in movie_paths:
+        for media in Media.objects:
-            if not (movie_path / "metadata.json").exists():
+            movies.append(build_movie_object(media))
-                match = movie_name_and_year.match(movie_path.name)
-                if not match:
-                    mobj = {
-                        "title":movie_path.name,
-                        "title_english":movie_path.name,
-                        "title_long":movie_path.name,
-                        "state":"ok"
-                    }
-                else:
-                    movie_name, movie_year = match.groups()
-                    mobj = {
-                        "title":movie_name,
-                        "title_english":movie_name,
-                        "title_long":movie_path.name,
-                        "year":movie_year,
-                        "state": "ok"
-                    }
-                movies.append(mobj)
-                continue
-            with (movie_path / "metadata.json").open()  as f:
-                metadata = json.loads(f.read())
-                mobj = {
-                    "title":metadata["title"],
-                    "title_english":metadata["title"],
-                    "title_long":movie_path.name,
-                    "year":metadata["year"],
-                    "imdb_code": metadata["imdb_id"],
-                    "rating": metadata["rating"],
-                    "summary": metadata["plot_outline"],
-                    "synopsis": metadata["plot_outline"],
-                    "mpa_rating": metadata["certification"],
-                    "genres": metadata["genres"],
-                    "yt_trailer_code": metadata["yt_trailer_code"],
-                    "state": "ok"
-                }
-                try:
-                    metadata["runtime"] = int(metadata["runtime"]) // 100
-                except ValueError as err:
-                    pass
-                if metadata["plots"]:
-                    mobj["description_full"] = metadata["plots"][0]
-                movies.append(mobj)
         jobj = {"data":{
                     "limit":len(movies),
                     "count":len(movies),
@@ -73,23 +76,37 @@ class MoviesCollection(BaseMovieResource):
 class MoviesResource(BaseMovieResource):
 
     def on_get(self, req, resp, movie):
-        resp.json = [{"path": self.path, "movie":movie}]
+        try:
-
+            media = Media.objects.get(id=movie)
-class MovieStreamUrlResource(BaseMovieResource):
+        except ValidationError:
-    
+            media = Media.objects.get(imdb_id=movie)
-    def on_get(self, req, resp, movie):
+        resp.json = build_movie_object(media)
-        movie_name = movie
-        path = self.path / "Filmid"
-        movie_files = os.listdir(str(path / movie_name))
-        sizes = []
-        for movie_file in movie_files:
-            sizes.append((os.path.getsize(str(path / movie_name / movie_file)), movie_file))
-        file_name = sorted(sizes)[-1][1]
-        resp.json = {
-            "default": 'https://media.arti.ee/Filmid/{}/{}'.format(movie_name.replace(" ", "%20"), file_name.replace(" ", "%20"))
-        }
 
 class MagnetResource():
 
     def on_post(self, req, resp):
-        resp.json = [req.json]
+        json = req.json
+        if not 'info_hash' in json:
+            raise ValueError("info_hash missing from request")
+        task = TaskMetainfoDl(info_hash=json['info_hash'])
+        task.imdb_code = json.get('imdb_code')
+        task.save()
+        tasks.metainfo_dl(str(task.id))
+        resp.json = {"task_id":task.id}
+
+
+class TaskCollection():
+
+    def on_get(self, req, resp):
+        tasks = list()
+        for task in Task.objects:
+            tasks.append(task.to_mongo())
+        resp.json = tasks
+
+class TaskResource():
+    def on_get(self, req, resp, task_id):
+        task = Task.objects.get(id=task_id)
+        resp.json = task.to_mongo()
+
+    def on_delete(self, req, resp, task_id):
+        Task.objects(id=task_id).delete()

api/karumedia/models.py

@@ -0,0 +1,54 @@
+import datetime
+from mongoengine import *
+
+class Media(Document):
+    title = StringField()
+    slug = StringField(unique=True)
+    imdb_id = StringField(unique=True)
+    runtime = IntField()
+    genres = ListField(StringField())
+    mpa_rating = StringField()
+    rating = FloatField()
+    yt_trailer_code = StringField()
+    date_added = DateTimeField(default=datetime.datetime.now)
+    parrent = ReferenceField('self')
+    season = IntField()
+    episode = IntField()
+    release_date = StringField()
+    type = StringField()
+    plot_outline = StringField()
+    plots = ListField(StringField())
+    year = IntField()
+
+class MediaFile(Document):
+    url = URLField()
+    path = StringField()
+    media = ReferenceField(Media)
+    mimetype = StringField()
+    resolution = StringField(choices=('480p', '720p', '1080p', 'native'), default="native")
+
+class MediaPoster(Document):
+    poster = FileField()
+    media = ReferenceField(Media)
+
+class Task(Document):
+    progress = FloatField(default=0)
+    state = StringField(choices=('starting', 'running', 'done'), default='starting')
+    creation_time = DateTimeField(default=datetime.datetime.now)
+    sub_tasks = ListField(ReferenceField('self'), default=list)
+
+    meta = {'allow_inheritance': True}
+
+class TaskMetainfoDl(Task):
+    info_hash = StringField(required=True)
+    imdb_id = StringField()
+
+class TaskImdbDl(Task):
+    imdb_id = StringField(required=True)
+    media = ReferenceField(Media)
+
+class TaskTorrentDl(Task):
+    info_hash = StringField(required=True)
+    imdb_id = StringField()
+    media = ReferenceField(Media)
+

api/karumedia/tasks.py

@@ -0,0 +1,281 @@
+from uwsgi_tasks import *
+from .models import *
+from time import sleep
+from requests import get
+import transmissionrpc
+from pprint import pprint
+from base64 import b64decode
+from urllib.parse import urlencode, quote
+import PTN
+from imdbpie import Imdb
+import os
+import subprocess
+import requests
+import magic
+import json
+
+YTKEY = os.environ.get("YTKEY")
+if not YTKEY:
+    error("YTKEY not set")
+    exit(1)
+
+
+default_trackers = [
+    'udp://glotorrents.pw:6969/announce',
+    'udp://tracker.openbittorrent.com:80',
+    'udp://tracker.coppersurfer.tk:6969',
+    'udp://tracker.leechers-paradise.org:6969',
+    'udp://p4p.arenabg.ch:1337',
+    'udp://tracker.internetwarriors.net:1337',
+    'udp://tracker.opentrackr.org:1337/announce'
+]
+
+r = get("https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt")
+
+best_trackers = r.text
+
+for tracker in best_trackers.split("\n"):
+    tracker = tracker.strip()
+    if tracker:
+        default_trackers.append(tracker)
+
+def hash_to_magnet(infoHash, name=None, trackers=None):
+    try:
+        b64decode(infoHash)
+    except:
+        raise Exception("Invalid infoHash")
+    magnet = {
+        "dn": name,
+        "tr": list(default_trackers)
+    }
+    if not name:
+        del magnet["dn"]
+    if trackers:
+        magnet["tr"].extend(trackers)
+    return "magnet:?xt=urn:btih:{}&".format(infoHash) + urlencode(magnet, doseq=True)
+
+
+
+tc = transmissionrpc.Client("172.20.1.2", user="admin", password="minemetsa")
+imdb = Imdb()
+
+def guess_imdb_code(title):
+    info = PTN.parse(title)
+    if 'year' not in info:
+        print("No title year found in title")
+    results = imdb.search_for_title(info["title"])
+    if not results:
+        return None
+    if 'year' in info:
+        match = [movie for movie in results if movie["year"] == str(info["year"])]
+        if not match:
+            pprint(results)
+            return None
+        else:
+            match = match[0]["imdb_id"]
+    else:
+        match = results[0]["imdb_id"]
+    return match
+
+
+@task
+def metainfo_dl(task_id):
+    task = TaskMetainfoDl.objects.get(id=task_id)
+
+    magnet = hash_to_magnet(task.info_hash)
+    t = tc.add_torrent(magnet)
+    print(task.info_hash.lower())
+    print(t.hashString)
+
+    task.state = "running"
+    task.save()
+
+    while True:
+        t = tc.get_torrent(t.hashString)
+        print(t.name, t.status, t.metadataPercentComplete*100, t.percentDone*100)
+        if t.metadataPercentComplete == 1:
+            break
+        task.progress = t.metadataPercentComplete*100
+        task.save()
+        sleep(1)
+
+    pprint(t.files())
+    t.stop()
+
+    if not task.imdb_id:
+        imdb_id = guess_imdb_code(t.name)
+    else:
+        imdb_id = task.imdb_id
+    print("imdb_id:", imdb_id)
+
+    try:
+        media = Media.objects.get(imdb_id=imdb_id)
+        print("Found existing media object:", media)
+    except DoesNotExist:
+        media = Media()
+        media.imdb_id = imdb_id
+        media.save()
+        print("Created a new media object:", media, media.imdb_id)
+
+    imdb_dl_task = TaskImdbDl(imdb_id=imdb_id)
+    imdb_dl_task.media = media
+    imdb_dl_task.save()
+    imdb_dl(str(imdb_dl_task.id))
+
+    torrent_dl_task = TaskTorrentDl(info_hash=t.hashString, imdb_id=imdb_id)
+    torrent_dl_task.media = media
+    torrent_dl_task.save()
+    torrent_dl(str(torrent_dl_task.id))
+
+    task.state = "done"
+    task.sub_tasks.append(imdb_dl_task)
+    task.sub_tasks.append(torrent_dl_task)
+    task.media = media
+    task.save()
+
+@task
+def imdb_dl(task_id):
+    task = TaskImdbDl.objects.get(id=task_id)
+    task.state = "running"
+    task.save()
+    print("imdb_id:", task.imdb_id)
+    try:
+        media = Media.objects.get(imdb_id=task.imdb_id)
+        print("Using existing media object:", media)
+    except DoesNotExist:
+        media = Media()
+        print("Creating a new media object")
+    media.imdb_id = task.imdb_id
+
+    title = imdb.get_title_by_id(task.imdb_id)
+    media.title = title.title
+    media.year = title.year
+    media.runtime = int(title.runtime)
+    media.genres = title.genres
+    media.mpa_rating = title.certification
+    media.release_date = title.release_date
+    media.type = title.type
+    media.plot_outline = title.plot_outline
+    media.plots = title.plots
+    media.rating = title.rating
+    media.save()
+
+    task.media = media
+    task.save()
+
+
+    params = {"key": YTKEY,
+              "part":"id", "maxResults":1,
+              "q":"{} ({}) trailer".format(media.title, media.year)}
+    r = requests.get("https://www.googleapis.com/youtube/v3/search", params=params)
+    media.yt_trailer_code = r.json()["items"][0]["id"]["videoId"]
+    media.save()
+
+    task.state = "done"
+    task.progress = 100
+    task.save()
+
+
+def probe(vid_file_path):
+    ''' Give a json from ffprobe command line
+
+    @vid_file_path : The absolute (full) path of the video file, string.
+    '''
+    if type(vid_file_path) != str:
+        raise Exception('Gvie ffprobe a full file path of the video')
+        return
+
+    command = ["ffprobe",
+            "-loglevel",  "quiet",
+            "-print_format", "json",
+             "-show_format",
+             "-show_streams",
+             vid_file_path
+             ]
+
+    pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    out, err = pipe.communicate()
+    return json.loads(out)
+
+
+def duration(vid_file_path):
+    ''' Video's duration in seconds, return a float number
+    '''
+    _json = probe(vid_file_path)
+    pprint(_json)
+
+    if 'format' in _json:
+        if 'duration' in _json['format']:
+            return float(_json['format']['duration'])
+
+    if 'streams' in _json:
+        # commonly stream 0 is the video
+        for s in _json['streams']:
+            if 'duration' in s:
+                return float(s['duration'])
+
+    # if everything didn't happen,
+    # we got here because no single 'return' in the above happen.
+    raise Exception('I found no duration')
+    #return None
+
+@task
+def torrent_dl(task_id):
+    task = TaskTorrentDl.objects.get(id=task_id)
+    task.state = "running"
+    task.save()
+
+    t = tc.get_torrent(task.info_hash)
+    t.start()
+
+    while True:
+        t = tc.get_torrent(task.info_hash)
+        print(t.name, t.status, t.metadataPercentComplete*100, t.percentDone*100)
+        if t.percentDone == 1:
+            break
+        task.progress = t.percentDone*100
+        task.save()
+        sleep(1)
+
+    if "imdb_id" in task:
+        try:
+            media = Media.objects.get(imdb_id=task.imdb_id)
+        except DoesNotExist:
+            pass
+        else:
+            tc.rename_torrent_path(task.info_hash, t.name, f"{media.title} ({media.year})")
+
+    tc.move_torrent_data(task.info_hash, "/srv/media/Filmid")
+
+    with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m:
+        for file_id, file_data in t.files().items():
+            file_path = "/srv/media/Filmid/"+file_data["name"]
+            file_type = m.id_filename(file_path)
+            video_duration = None
+            if file_type.startswith("video"):
+                video_duration = duration(file_path)
+
+            print(file_path, file_type, video_duration)
+            if not MediaFile.objects(path = file_path):
+                media_file = MediaFile()
+                media_file.media = Media.objects.get(imdb_id=task.imdb_id)
+                media_file.path = file_path
+                media_file.url = f"http://media.arti.ee/Filmid/{quote(file_data['name'])}"
+                media_file.mimetype = file_type
+                media_file.resolution = "native"
+                media_file.save()
+
+    media = task.media
+    cover_path = f"/srv/media/Filmid/{media.title} ({media.year})/cover.jpg"
+    if not MediaFile.objects(media=media, path=cover_path):
+        subprocess.call(['wget', imdb.get_title_by_id(media.imdb_id).cover_url,
+                         "-O", cover_path])
+        media_poster = MediaFile()
+        media_poster.path = cover_path
+        movie_name_and_year = quote(f"{media.title} ({media.year})")
+        media_poster.url = f"https://media.arti.ee/Filmid/{movie_name_and_year}/cover.jpg"
+        media_poster.save()
+
+    task.state = "done"
+    task.progress = 100
+    task.save()

api/karumedia/tools.py

@@ -1,10 +1,24 @@
 
 import json
+import datetime
 
 from falcon import Request as FalconRequest
 from falcon import Response as FalconResponse
-from falcon.errors import HTTPBadRequest, HTTPMissingParam, HTTPError
+from falcon.errors import HTTPBadRequest, HTTPMissingParam, HTTPError, HTTPNotFound
 import falcon.status_codes as status
+from bson.objectid import ObjectId
+from mongoengine import DoesNotExist
+
+class BSONDumps(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, ObjectId):
+            return str(obj)
+        elif isinstance(obj, datetime.datetime):
+            return obj.timestamp()
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)
+
+dumps = BSONDumps(indent=4).encode
 
 
 class JsonRequest(FalconRequest):
@@ -37,9 +51,12 @@ class JsonResponse(FalconResponse):
     @json.setter
     def json(self, value):
         self._json = value
-        self.body = json.dumps(value, indent=4)
+        self.body = dumps(value)
 
 def error_handler(ex, req, resp, params):
+    if type(ex).__name__ == DoesNotExist.__name__:
+        raise HTTPNotFound(title=type(ex).__name__, description=str(ex))
+    else:
         raise HTTPBadRequest(type(ex).__name__, str(ex))
 
 class TODOException(HTTPError):

api/requirements.txt

@@ -1 +1,13 @@
-falcon
+CacheControl==0.11.7
+falcon==1.1.0
+filemagic==1.6
+imdbpie==4.2.0
+lockfile==0.12.2
+mongoengine==0.11.0
+parse-torrent-name==1.1.1
+pymongo==3.4.0
+python-mimeparse==1.6.0
+requests==2.12.5
+six==1.10.0
+transmissionrpc==0.11
+uwsgi-tasks==0.6.4

api/uwsgi.ini

@@ -14,6 +14,9 @@ http=:8080
 #static-index=index.html
 mount=/api/v1=wsgi.py
 manage-script-name = true
+py-autoreload = 1
+spooler = taskspool
+spooler-processes = 3
 
 
 [prod]