chx_compress_working.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XPCS&XSVS Pipeline for Setup of (Gi)-SAXS\n",
    "\"This notebook corresponds to version {{ version }} of the pipeline tool: https://github.com/NSLS-II/pipelines\"\n",
    "\n",
    "This notebook begins with a raw time-series of images and ends with $g_2(t)$ for a range of $q$, fit to an exponential or stretched exponential, and a two-time correlation functoin.\n",
    "\n",
    "## Overview\n",
    "\n",
    "* Setup: load packages/setup path\n",
    "* Load Metadata & Image Data\n",
    "* Apply Mask\n",
    "* Compress Data\n",
    "* Define Q-ROI, e.g. qr for SAXS, (qr, qz) for gi-saxs\n",
    "\n",
    "The important scientific code is imported from the [chxanalys](https://github.com/yugangzhang/chxanalys/tree/master/chxanalys) and [scikit-beam](https://github.com/scikit-beam/scikit-beam) project. Refer to chxanalys and scikit-beam for additional documentation and citation information.\n",
    "\n",
    "## CHX Olog NoteBook\n",
    "CHX Olog (https://logbook.nsls2.bnl.gov/11-ID/)\n",
    "\n",
    "## Load Packages\n",
    "\n",
    "Import packages for I/O, visualization, and analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyCHX.chx_packages import *\n",
    "%matplotlib notebook\n",
    "plt.rcParams.update({'figure.max_open_warning': 0})\n",
    "plt.rcParams.update({ 'image.origin': 'lower'   })\n",
    "plt.rcParams.update({ 'image.interpolation': 'none'   })\n",
    "import pickle as cpk\n",
    "from pyCHX.chx_xpcs_xsvs_jupyter_V1 import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Errno 2] No such file or directory: 'sample_data'\n",
      "/CENTRAL/xf11id1/analysis/2018_2/commisioning/Compress_Work/sample_data\n"
     ]
    }
   ],
   "source": [
    "cd sample_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run ~/Repos/chx_xpcs/chx_xpcs/io/multifile/multifile.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = 'sample_compress.bin'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing file...\n",
      "Done. Took 0.030787944793701172 secs for 1000 frames\n"
     ]
    }
   ],
   "source": [
    "f = MultifileBNL(fp )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.1 ms ± 2.7 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "f.rdframe(0)\n",
    "#f.rdframe(1000  )\n",
    "f.rdframe(1000 -1 )\n",
    "f.rdframe(500 -1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyCHX.chx_packages import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'MultifileL' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m----------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-f7b6447b92ad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mFD\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMultifileL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'MultifileL' is not defined"
     ]
    }
   ],
   "source": [
    "FD = Multifile(fp )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.55 s ± 199 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "FD.rdframe(0)\n",
    "#f.rdframe(1000  )\n",
    "FD.rdframe(1000 -1 )\n",
    "FD.rdframe(500 -1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = '/home/yuzhang/analysis/Compressed_Data/uid_fefc3fac-af61-4248-be62-04cdfb231688.cmp'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing file...\n",
      "Done. Took 77.2925763130188 secs for 7200 frames\n"
     ]
    }
   ],
   "source": [
    "f = MultifileBNL(fp )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.3 ms ± 518 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "f.rdframe(0)\n",
    "#f.rdframe(1000  )\n",
    "f.rdframe(7200 -1 )\n",
    "f.rdframe(3600 -1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "FD = Multifile(fp, 0, 7200 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.24 s ± 35.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "FD.rdframe(0)\n",
    "#f.rdframe(1000  )\n",
    "FD.rdframe(7200 -1 )\n",
    "FD.rdframe(3600 -1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "294"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "3240//11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 1., 1., ..., 0., 1., 0.],\n",
       "       [2., 1., 0., ..., 2., 1., 0.],\n",
       "       [1., 2., 2., ..., 1., 2., 0.],\n",
       "       ...,\n",
       "       [1., 1., 1., ..., 1., 1., 1.],\n",
       "       [1., 1., 1., ..., 1., 1., 1.],\n",
       "       [0., 2., 0., ..., 1., 2., 0.]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FD.rdframe(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#(f.rdframe(7000).T - FD.rdframe(7000)).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For compress data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "uid = 'fefc3f'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filepath : /XF11ID/data/2018/02/15/016f0386-b6e2-4c0a-800d\n",
      "got images_per_file\n"
     ]
    }
   ],
   "source": [
    "sud = get_sid_filenames(db[uid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/XF11ID/data/2018/02/15/016f0386-b6e2-4c0a-800d_2701_master.h5\n"
     ]
    }
   ],
   "source": [
    "fp = sud[2][0]\n",
    "print(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     73      73    4959\r\n"
     ]
    }
   ],
   "source": [
    "!ls /XF11ID/data/2018/02/15/016f0386-b6e2-4c0a-800d_2701* |wc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!ls /XF11ID/data/2018/02/15/016f0386-b6e2-4c0a-800d_2701*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/CENTRAL/xf11id1/analysis/2018_2/commisioning/Compress_Work/sample_data'"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run ~/Repos/chx_xpcs/chx_xpcs/io/eiger/compress_file.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading dataset 1\n",
      "reading dataset 2\n",
      "reading dataset 3\n",
      "reading dataset 4\n",
      "reading dataset 5\n",
      "reading dataset 6\n",
      "reading dataset 7\n",
      "reading dataset 8\n",
      "reading dataset 9\n",
      "reading dataset 10\n",
      "reading dataset 11\n",
      "reading dataset 12\n",
      "reading dataset 13\n",
      "reading dataset 14\n",
      "reading dataset 15\n",
      "reading dataset 16\n",
      "reading dataset 17\n",
      "reading dataset 18\n",
      "reading dataset 19\n",
      "reading dataset 20\n",
      "reading dataset 21\n",
      "reading dataset 22\n",
      "reading dataset 23\n",
      "reading dataset 24\n",
      "reading dataset 25\n",
      "reading dataset 26\n",
      "reading dataset 27\n",
      "reading dataset 28\n",
      "reading dataset 29\n",
      "reading dataset 30\n",
      "reading dataset 31\n",
      "reading dataset 32\n",
      "reading dataset 33\n",
      "reading dataset 34\n",
      "reading dataset 35\n",
      "reading dataset 36\n",
      "reading dataset 37\n",
      "reading dataset 38\n",
      "reading dataset 39\n",
      "reading dataset 40\n",
      "reading dataset 41\n",
      "reading dataset 42\n",
      "reading dataset 43\n",
      "reading dataset 44\n",
      "reading dataset 45\n",
      "reading dataset 46\n",
      "reading dataset 47\n",
      "reading dataset 48\n",
      "reading dataset 49\n",
      "reading dataset 50\n",
      "reading dataset 51\n",
      "reading dataset 52\n",
      "reading dataset 53\n",
      "reading dataset 54\n",
      "reading dataset 55\n",
      "reading dataset 56\n",
      "reading dataset 57\n",
      "reading dataset 58\n",
      "reading dataset 59\n",
      "reading dataset 60\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "Can't read data (file read failed: time = Fri Jun  8 12:04:32 2018\n, filename = '/XF11ID/data/2018/02/15/016f0386-b6e2-4c0a-800d_2701_data_000060.h5', file descriptor = 56, errno = 116, error message = 'Stale file handle', buf = 0x564fbfcb1d08, total read size = 1048576, bytes this sub-read = 1048576, bytes actually read = 18446744073709551615, offset = 242384336)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m----------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<timed eval>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/Repos/chx_xpcs/chx_xpcs/io/eiger/compress_file.py\u001b[0m in \u001b[0;36mcompress_file\u001b[0;34m(filename, dset_min, dset_max, dset_root, dset_pref, outfile)\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdset_root\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdset_pref\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"{:06d}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0mdset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0mdset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_direct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ms_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0;31m#test = np.array(f[key][j])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/chx_compress/lib/python3.6/site-packages/h5py/_hl/dataset.py\u001b[0m in \u001b[0;36mread_direct\u001b[0;34m(self, dest, source_sel, dest_sel)\u001b[0m\n\u001b[1;32m    655\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    656\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mmspace\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdest_sel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbroadcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_sel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 657\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmspace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfspace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdxpl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dxpl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    658\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    659\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwrite_direct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource_sel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdest_sel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/h5d.pyx\u001b[0m in \u001b[0;36mh5py.h5d.DatasetID.read\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/_proxy.pyx\u001b[0m in \u001b[0;36mh5py._proxy.dset_rw\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/_proxy.pyx\u001b[0m in \u001b[0;36mh5py._proxy.H5PY_H5Dread\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: Can't read data (file read failed: time = Fri Jun  8 12:04:32 2018\n, filename = '/XF11ID/data/2018/02/15/016f0386-b6e2-4c0a-800d_2701_data_000060.h5', file descriptor = 56, errno = 116, error message = 'Stale file handle', buf = 0x564fbfcb1d08, total read size = 1048576, bytes this sub-read = 1048576, bytes actually read = 18446744073709551615, offset = 242384336)"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "compress_file(fp, dset_min=1, dset_max=72, dset_root=\"/entry/data\",\n",
    "        dset_pref=\"data_\", outfile=\"uid=%s.bin\"%uid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pyCHX_Comp",
   "language": "python",
   "name": "pychx_comp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}