meteortools.ukmondb.getLiveImages

  1# Copyright (C) 2018-2023 Mark McIntyre
  2#
  3# python script to get all live JPGs for a specified time
  4#
  5import os
  6import pandas as pd
  7import requests
  8import datetime
  9
 10
 11def getLiveJpgs(dtstr, outdir=None, create_txt=False):
 12    """
 13    Retrieve live images from the ukmon website that match a pattern  
 14
 15    Arguments:  
 16        dtstr:      [str] Date in YYYYMMDD_HHMMSS format. Partial strings allowed  
 17        outdir:     [str] Where to save the file. Default is to create a folder named dtstr  
 18        create_txt: [bool] If true, create a text file containing the pattern matches  
 19
 20    Notes:  
 21        We only keep the last few thousand live images so this function will return nothing
 22        for older data. 
 23    """
 24    if outdir is None:
 25        outdir = dtstr
 26    os.makedirs(outdir, exist_ok=True)
 27
 28    apiurl = 'https://api.ukmeteors.co.uk/liveimages/getlive'
 29
 30    while len(dtstr) < 15:
 31        dtstr = dtstr + '0'    
 32    isodt1 = datetime.datetime.strptime(dtstr[:15],'%Y%m%d_%H%M%S')
 33    fromdstr = isodt1.isoformat()[:19]+'.000Z'
 34    isodt2 = isodt1 + datetime.timedelta(minutes=1)
 35    todstr = isodt2.isoformat()[:19]+'.000Z'
 36    liveimgs = pd.read_json(f'{apiurl}?dtstr={fromdstr}&enddtstr={todstr}&fmt=withxml')
 37
 38    for _, img in liveimgs.iterrows():
 39        try:
 40            jpgurl = img.urls['url']
 41            fname = _download(jpgurl, outdir)
 42            print(f'retrieved {fname}')
 43            if create_txt:
 44                createTxtFile(img.image_name, outdir)
 45        except:
 46            print(f'{img.image_name} unavailable')
 47
 48
 49def getLiveimageList(dtstr):
 50    """ 
 51    Get a list URLs of livestream images matching a pattern YYYYMMDD_HHMMSS.  
 52    The seconds and minutes parts are optional but huge amounts of data may get returned.  
 53
 54    Note that we only keep the last month of images and so this function won't return
 55    anything for older dates. 
 56    Note also that the URLs are presigned and valid only for five minutes.  
 57
 58    Example pattern: '20230421_2122'
 59    """
 60    apiurl = 'https://api.ukmeteors.co.uk/liveimages/getlive'
 61    liveimgs = pd.read_json(f'{apiurl}?pattern={dtstr}')
 62    return liveimgs
 63
 64
 65def getFBfiles(patt, outdir=None):
 66    """
 67    Retrieve fireball files from the ukmon website that match a pattern  
 68
 69    Arguments:  
 70        patt:      [str] pattern to match.  
 71        outdir:     [str] Where to save the files. See notes.  
 72
 73    Returns:  
 74        a pandas dataframe containing the filenames and presigned URLs
 75
 76    Example:  
 77        import pandas as pd  
 78        df = getFBfiles('UK0006_20230421_2122', 'c:/temp/uk0006')  
 79
 80    Notes:  
 81        The function retrieves the FF and FR files matching the pattern, plus the config and platepar 
 82        for the camera, provided the files are available on our server.  
 83        If outdir is not supplied, a folder will be created in the current working directory named
 84        using the station ID code. 
 85    """
 86    if outdir is None:
 87        outdir = patt[:6]
 88    os.makedirs(outdir, exist_ok=True)
 89    apiurl = 'https://api.ukmeteors.co.uk/fireballfiles'
 90    fbfiles = pd.read_json(f'{apiurl}?pattern={patt}')
 91    if len(fbfiles) == 0:
 92        print('no matching data found')
 93        return None
 94    for _,fil in fbfiles.iterrows():
 95        fname = fil['filename']
 96        url = fil['url']
 97        _ = _download(url, outdir, fname)
 98        print(fname)
 99    return fbfiles
100
101
102def createTxtFile(fname, outdir=None):
103    """
104    Create a text file named after the cameraID, containing a list of fireball files 
105    to be retrieved from a remote camera.  
106
107    Arguments:  
108        fname:  [str] the name of the FF file to be retrieved. 
109        outdir: [str] where to save the files. See notes.  
110
111    Notes:  
112        the fname parameter should be the name of the live JPG for which you wish to 
113        retrieve the corresponding FF and FR files.  
114        If outdir is not supplied, the files will be saved in the current directory.   
115    """
116    if fname[0] == 'M':
117        spls = fname.split('_')
118        stationid = spls[-1][:6].lower()
119        dtime = fname[1:16]
120        patt = f'FF_{stationid}_{dtime}'
121        stationid = stationid.lower()
122    else:
123        patt = fname[:25]
124        stationid = fname[3:9].lower()
125    if outdir is None:
126        outdir = '.' 
127    os.makedirs(outdir, exist_ok=True)
128    
129    txtf = os.path.join(outdir, f'{stationid}.txt')
130    if os.path.isfile(txtf):
131        os.remove(txtf)
132    patt = patt.upper()
133    with open(txtf,'w') as outf:
134        outf.write(f'{patt}\n{patt.replace("FF_", "FR_")}\n')
135    return txtf
136
137
138def _download(url, outdir, fname=None):
139    get_response = requests.get(url, stream=True)
140    if fname is None:
141        fname = url.split('?')[0].split("/")[-1]
142    with open(os.path.join(outdir, fname), 'wb') as f:
143        for chunk in get_response.iter_content(chunk_size=4096):
144            if chunk: # filter out keep-alive new chunks
145                f.write(chunk)
146    return fname