#!/usr/bin/env python3 """NAEJOBS Data - sync script (naejobs.com/data.html) Keeps your own copy of the NAEJOBS Data file up to date. Each run asks NAEJOBS whether a new version has been published; only when there is one does it download it. Run it once a day, early morning (NAEJOBS publishes in the evening, US Pacific time); a download only happens on the days NAEJOBS has published a new version (about three times a week). Setup (once): 1. Put this file in a folder on your server. 2. Put your API key (it starts with njd_) in a file named naejobs-key.txt in the same folder, or set the NAEJOBS_KEY environment variable. 3. Run it once to try it: python3 naejobs-sync.py (Windows: py naejobs-sync.py) 4. Schedule it once a day (5 AM here): Mac / Linux (crontab -e): 0 5 * * * python3 /path/to/naejobs-sync.py >> /path/to/naejobs-sync.log 2>&1 Windows (Command Prompt): schtasks /Create /SC DAILY /ST 05:00 /TN "NAEJOBS sync" /TR "py C:\\path\\to\\naejobs-sync.py" What it leaves in the folder: naejobs-full.json the current file, unzipped, ready to load (always a complete file - it is swapped in only when whole) naejobs-full.json.gz the same file as downloaded naejobs-state.json the version you have (the script's memory - leave it there) Your file follows the filter saved on your account (naejobs.com/data.html, signed in -> Filter my file): only the matching postings and their employers, or the whole board with no filter. Changing the filter makes a new version of your file, and the next run downloads it. Options (environment variables, all optional): NAEJOBS_KEY your API key (instead of naejobs-key.txt) NAEJOBS_DIR where the files go (default: the folder this script is in) NAEJOBS_FORMAT json (default) or csv -> naejobs-full.csv / naejobs-full.csv.gz NAEJOBS_AFTER a command to run after a new version lands, e.g. your import: NAEJOBS_AFTER="php import.php" Exit codes: 0 = up to date or updated, 1 = error (the next run tries again), 2 = no downloads left for this version. Python 3.6 or newer, nothing to install. Keep your key on your server, never in browser code. """ import gzip import json import os import shutil import subprocess import sys import time import urllib.error import urllib.request API = os.environ.get('NAEJOBS_API', 'https://api.naejobs.com/api').rstrip('/') HERE = os.path.dirname(os.path.abspath(__file__)) DIR = os.path.abspath(os.environ.get('NAEJOBS_DIR') or HERE) FMT = 'csv' if os.environ.get('NAEJOBS_FORMAT', '').strip().lower() == 'csv' else 'json' STATE = os.path.join(DIR, 'naejobs-state.json') OUT = os.path.join(DIR, 'naejobs-full.' + FMT) UA = 'naejobs-sync/1.1' # 1.1 (datafilter, 2026-11-09): a changed saved filter = the same publishedAt with a new version -> downloaded def log(msg): print(time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + msg, flush=True) def read_key(): k = os.environ.get('NAEJOBS_KEY', '').strip() if not k: for folder in (DIR, HERE): p = os.path.join(folder, 'naejobs-key.txt') if os.path.isfile(p): raw = open(p, 'rb').read() # a key file saved by Windows PowerShell's "echo ... > file" is UTF-16; Notepad may add a BOM if raw[:2] in (b'\xff\xfe', b'\xfe\xff'): k = raw.decode('utf-16') else: k = raw.decode('utf-8-sig', 'replace') k = k.strip().strip('"').strip() break return k def load_state(): try: with open(STATE, 'r', encoding='utf-8') as f: s = json.load(f) return s if isinstance(s, dict) else {} except (OSError, ValueError): return {} def save_state(s): tmp = STATE + '.tmp' with open(tmp, 'w', encoding='utf-8') as f: json.dump(s, f, indent=2) os.replace(tmp, STATE) def call(path, key, etag=None, timeout=60): req = urllib.request.Request(API + path, headers={'X-Api-Key': key, 'User-Agent': UA, 'Accept': 'application/json'}) if etag: req.add_header('If-None-Match', etag) return urllib.request.urlopen(req, timeout=timeout) def err_text(e): try: body = e.read().decode('utf-8', 'replace') return json.loads(body).get('error') or body[:300] except Exception: return str(e) def older(a, b): """True when publishedAt a is EARLIER than b (ISO dates compare as text); unknown = not older. datafilter (2026-11-09): the SAME publishedAt with a new version is the same NAEJOBS publish with a changed saved filter (naejobs.com/data.html) - that one is downloaded; only a strictly older publish (an API instance still on the last board) is skipped.""" return bool(a) and bool(b) and str(a) < str(b) def file_head(path): """The version and publishedAt written at the top of a downloaded JSON file.""" try: with gzip.open(path, 'rt', encoding='utf-8-sig') as f: head = f.read(2000) i = head.find(',"employers"') h = json.loads(head[:i] + '}') if i > 0 else {} return h.get('version'), h.get('publishedAt') except Exception: return None, None def run_after(st): """Your import (NAEJOBS_AFTER). If it fails, the next run tries it again even when no new version came.""" after = os.environ.get('NAEJOBS_AFTER', '').strip() if not after: return 0 log('Running NAEJOBS_AFTER: ' + after) rc = subprocess.call(after, shell=True, cwd=DIR) st[FMT]['afterOk'] = rc == 0 save_state(st) if rc != 0: log('NAEJOBS_AFTER ended with code ' + str(rc) + ' - it runs again next time.') return 1 return 0 def main(): key = read_key() if not key.startswith('njd_'): log('No API key. Put your key (it starts with njd_) in naejobs-key.txt next to this script, or set NAEJOBS_KEY.') return 1 os.makedirs(DIR, exist_ok=True) st = load_state() have = st.get(FMT) or {} have_file = os.path.isfile(OUT) # 1. is there a new version? (free: answers "304 not modified" when nothing changed) try: r = call('/data/manifest', key, etag=('"' + have['version'] + '"') if have.get('version') and have_file else None) man = json.loads(r.read().decode('utf-8')) except urllib.error.HTTPError as e: if e.code == 304: log('Up to date (version ' + have.get('version', '?') + ', published ' + str(have.get('publishedAt')) + ').') return run_after(st) if have.get('afterOk') is False else 0 log('Could not check for a new version (' + str(e.code) + '): ' + err_text(e)) return 1 except Exception as e: log('Could not reach NAEJOBS: ' + str(e)) return 1 ver, pub = man.get('version'), man.get('publishedAt') if have_file and ver == have.get('version'): log('Up to date (version ' + str(ver) + ').') return run_after(st) if have.get('afterOk') is False else 0 if have_file and older(pub, have.get('publishedAt')): log('NAEJOBS answered with an older version (' + str(pub) + ') than yours (' + str(have.get('publishedAt')) + ') - keeping yours.') return 0 if man.get('downloadsLeft') == 0: log('A new version is out (' + str(pub) + ') but this account has no downloads left for it. It will try again next run.') return 2 # 2. download it to a temporary file, check it, then swap it in filt = man.get('filter') or {} log('New version ' + str(ver) + ' (published ' + str(pub) + ', ' + str(man.get('postings')) + ' postings' + ((', your filter: ' + str(filt.get('summary')).encode('ascii', 'replace').decode('ascii')) if filt else '') + ') - downloading ...') gz_tmp, out_tmp = OUT + '.gz.part', OUT + '.part' try: r = None for attempt in (1, 2): try: r = call('/data/latest?format=' + FMT, key, timeout=300) break except urllib.error.HTTPError as e: if e.code == 503 and attempt == 1: # NAEJOBS is loading a new board - give it a minute log('NAEJOBS is loading - trying again in 60 seconds ...') time.sleep(60) continue if e.code == 429: log('No downloads left for this version: ' + err_text(e)) return 2 log('Download failed (' + str(e.code) + '): ' + err_text(e)) return 1 got_ver = r.headers.get('X-NAEJOBS-Version') or ver left = r.headers.get('X-NAEJOBS-Downloads-Left') with open(gz_tmp, 'wb') as f: shutil.copyfileobj(r, f, 1 << 20) with open(gz_tmp, 'rb') as f: if f.read(2) != b'\x1f\x8b': raise ValueError('the download is not a gzip file') with gzip.open(gz_tmp, 'rb') as src, open(out_tmp, 'wb') as dst: # reading it all checks the gzip CRC: a cut-off download fails here shutil.copyfileobj(src, dst, 1 << 20) size = os.path.getsize(out_tmp) if size < 1000: raise ValueError('the file is too small (' + str(size) + ' bytes)') if FMT == 'json': fv, fp = file_head(gz_tmp) if fp: pub = fp if fv: got_ver = fv with open(out_tmp, 'rb') as f: f.seek(-200, os.SEEK_END) if not f.read().rstrip().endswith(b'}'): raise ValueError('the JSON file is not complete') if have_file and older(pub, have.get('publishedAt')): log('The download is older than your copy (' + str(pub) + ') - keeping yours.') os.remove(gz_tmp) os.remove(out_tmp) return 0 except Exception as e: log('Download failed: ' + str(e)) for p in (gz_tmp, out_tmp): try: os.remove(p) except OSError: pass return 1 try: os.replace(gz_tmp, OUT + '.gz') os.replace(out_tmp, OUT) except OSError as e: # e.g. on Windows, another program has naejobs-full.json open log('Could not swap in the new file (' + str(e) + ') - is another program holding it open? The next run tries again.') return 1 st[FMT] = {'version': got_ver, 'publishedAt': pub, 'downloadedAt': time.strftime('%Y-%m-%dT%H:%M:%S'), 'bytes': size} save_state(st) log('Updated: ' + OUT + ' (' + format(size, ',') + ' bytes, version ' + str(got_ver) + (', ' + left + ' downloads left for it' if left else '') + ').') # 3. your import, if you set one return run_after(st) if __name__ == '__main__': sys.exit(main())