<!-- 
RSS generated by JIRA (8.3.4#803005-sha1:1f96e09b3c60279a408a2ae47be3c745f571388b) at Sat Feb 10 16:03:54 JST 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>PFS-JIRA</title>
    <link>https://pfspipe.ipmu.jp/jira</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>8.3.4</version>
        <build-number>803005</build-number>
        <build-date>13-09-2019</build-date>
    </build-info>


<item>
            <title>[PIPE2D-1180] profile isrTask reducing n arm</title>
                <link>https://pfspipe.ipmu.jp/jira/browse/PIPE2D-1180</link>
                <project id="10002" key="PIPE2D">DRP 2-D Pipeline</project>
                    <description>&lt;p&gt;When reducing n2 data at LAM, we noticed that it takes quite a long time (~2min30) to get an postISR image, please profile the current code and try to find improvement that would shorten that time.&lt;/p&gt;

&lt;p&gt;FYI, that how we get those images :&lt;/p&gt;

&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;import numpy as np
from lsst.obs.pfs.isrTask import PfsIsrTask as IsrTask

config = IsrTask.ConfigClass()
config.doBias = False
config.doDark = False
config.doFringe = False
config.doFlat = False
config.doLinearize = False
config.doDefect = True
config.doIPC = True
config.ipcCoeffs = np.array([13e-3, 6e-3])
config.doSaturationInterpolation = False
config.validate()

isrTask = IsrTask(config=config)

raw = butler.get(&apos;raw&apos;, dataId=dataId)
defects = butler.get(&apos;defects&apos;, dataId)
calexp = isrTask.run(raw, defects=defects)
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</description>
                <environment></environment>
        <key id="23292">PIPE2D-1180</key>
            <summary>profile isrTask reducing n arm</summary>
                <type id="3" iconUrl="https://pfspipe.ipmu.jp/jira/secure/viewavatar?size=xsmall&amp;avatarId=10518&amp;avatarType=issuetype">Task</type>
                                            <priority id="10000" iconUrl="https://pfspipe.ipmu.jp/jira/images/icons/priorities/medium.svg">Normal</priority>
                        <status id="10002" iconUrl="https://pfspipe.ipmu.jp/jira/images/icons/statuses/generic.png" description="The issue is resolved, reviewed, and merged">Done</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10000">Done</resolution>
                                        <assignee username="arnaud.lefur">arnaud.lefur</assignee>
                                    <reporter username="arnaud.lefur">arnaud.lefur</reporter>
                        <labels>
                            <label>EngRun</label>
                    </labels>
                <created>Wed, 8 Mar 2023 01:00:55 +0000</created>
                <updated>Fri, 5 May 2023 03:09:52 +0000</updated>
                            <resolved>Fri, 5 May 2023 03:09:51 +0000</resolved>
                                                                        <due></due>
                            <votes>0</votes>
                                    <watches>4</watches>
                                                                <comments>
                            <comment id="32369" author="price" created="Wed, 8 Mar 2023 01:39:13 +0000"  >&lt;p&gt;Would you please run that script under &lt;tt&gt;python -m cProfile -o profile.dat&lt;/tt&gt;, and then post the result of running &lt;tt&gt;python -c &apos;from pstats import Stats; Stats(&quot;profile.dat&quot;).sort_stats(&quot;cumulative&quot;).print_stats(30)&apos;&lt;/tt&gt;?&lt;/p&gt;</comment>
                            <comment id="32377" author="arnaud.lefur" created="Thu, 9 Mar 2023 22:59:11 +0000"  >&lt;p&gt;Here&apos;s the results :&lt;/p&gt;
&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;Thu Mar  9 22:56:44 2023    profile.dat

         16660323 function calls (16092265 primitive calls) in 195.285 seconds

   Ordered by: cumulative time
   List reduced from 15070 to 30 due to restriction &amp;lt;30&amp;gt;

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
   3708/1    0.078    0.000  195.304  195.304 {built-in method builtins.exec}
        1    0.000    0.000  195.303  195.303 test_detrend_n2.py:1(&amp;lt;module&amp;gt;)
        1    0.000    0.000  155.743  155.743 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/obs_pfs/w.2023.09/python/lsst/obs/pfs/isrTask.py:422(run)
        1    0.037    0.037  155.743  155.743 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/obs_pfs/w.2023.09/python/lsst/obs/pfs/isrTask.py:550(runH4RG)
        1    0.197    0.197  148.283  148.283 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/isrTask.py:2497(maskAndInterpolateDefects)
        1    0.421    0.421  145.692  145.692 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/isrFunctions.py:157(interpolateFromMask)
        1    0.000    0.000  141.637  141.637 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/isrFunctions.py:77(interpolateDefectList)
        1  140.737  140.737  140.866  140.866 {built-in method lsst.meas.algorithms.interp.interpolateOverDefects}
        2    0.000    0.000   23.277   11.639 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/daf_persistence/g6a31054a6e+4ae6138134/python/lsst/daf/persistence/butler.py:1377(get)
        2    0.000    0.000   21.831   10.916 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/daf_persistence/g6a31054a6e+4ae6138134/python/lsst/daf/persistence/butler.py:1414(callback)
        3    0.000    0.000   21.279    7.093 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/daf_persistence/g6a31054a6e+4ae6138134/python/lsst/daf/persistence/posixStorage.py:262(read)
        1    0.000    0.000   21.279   21.279 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/daf_persistence/g6a31054a6e+4ae6138134/python/lsst/daf/persistence/butler.py:1409(callback)
        1    0.000    0.000   21.279   21.279 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/daf_persistence/g6a31054a6e+4ae6138134/python/lsst/daf/persistence/butler.py:1583(_read)
        1    0.000    0.000   21.279   21.279 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/daf_persistence/g6a31054a6e+4ae6138134/python/lsst/daf/persistence/repository.py:186(read)
        1    0.000    0.000   21.278   21.278 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/daf_persistence/g6a31054a6e+4ae6138134/python/lsst/daf/persistence/posixStorage.py:565(readFitsStorage)
        1    0.124    0.124   21.278   21.278 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/calibType.py:432(readFits)
        1    4.021    4.021   20.786   20.786 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/defects.py:492(fromTable)
      3/2    1.035    0.345   15.111    7.556 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/defects.py:83(__init__)
   3150/8    0.028    0.000   14.570    1.821 &amp;lt;frozen importlib._bootstrap&amp;gt;:986(_find_and_load)
   3132/5    0.022    0.000   14.570    2.914 &amp;lt;frozen importlib._bootstrap&amp;gt;:956(_find_and_load_unlocked)
   4050/4    0.006    0.000   14.556    3.639 &amp;lt;frozen importlib._bootstrap&amp;gt;:211(_call_with_frames_removed)
   3010/6    0.023    0.000   14.556    2.426 &amp;lt;frozen importlib._bootstrap&amp;gt;:650(_load_unlocked)
   2631/6    0.012    0.000   14.556    2.426 &amp;lt;frozen importlib._bootstrap_external&amp;gt;:837(exec_module)
  1063/54    0.004    0.000   14.030    0.260 {built-in method builtins.__import__}
        1    0.000    0.000   11.582   11.582 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/obs_pfs/w.2023.09/python/lsst/obs/pfs/__init__.py:1(&amp;lt;module&amp;gt;)
        1    0.000    0.000   10.386   10.386 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/obs_pfs/w.2023.09/python/lsst/obs/pfs/pfsMapper.py:1(&amp;lt;module&amp;gt;)
3620/1018    0.014    0.000   10.299    0.010 &amp;lt;frozen importlib._bootstrap&amp;gt;:1017(_handle_fromlist)
662458/441639    2.446    0.000    9.557    0.000 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/defects.py:174(_normalize)
        2    0.000    0.000    7.421    3.710 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/defects.py:641(fromFootprintList)
        3    7.180    2.393    7.180    2.393 /software/stack/20220609/stack/miniconda3-py38_4.9.2-3.0.0/Linux64/ip_isr/g8142d5ee96+96f555657d/python/lsst/ip/isr/defects.py:261(maskPixels)

&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</comment>
                            <comment id="32378" author="price" created="Thu, 9 Mar 2023 23:13:02 +0000"  >&lt;p&gt;So the tall pole is &lt;tt&gt;lsst.meas.algorithms.interp.interpolateOverDefects&lt;/tt&gt;. There are a lot of defects in the NIR detector, and that function is well-known to be relatively slow.&lt;/p&gt;

&lt;p&gt;One option for a significant speed boost would be to simply replace bad pixels with the image median, but it would not be as visually pleasing.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://pfspipe.ipmu.jp/jira/secure/ViewProfile.jspa?name=rhl&quot; class=&quot;user-hover&quot; rel=&quot;rhl&quot;&gt;rhl&lt;/a&gt; should advise.&lt;/p&gt;</comment>
                            <comment id="32527" author="arnaud.lefur" created="Thu, 6 Apr 2023 15:04:52 +0000"  >&lt;p&gt;We will be living with this, at least on short term, so we should probably close this ticket, unless someone wants to chime in.&lt;/p&gt;</comment>
                            <comment id="32529" author="cloomis" created="Thu, 6 Apr 2023 15:20:07 +0000"  >&lt;p&gt;How hard would it be to allow median interpolation? Given that it takes &amp;gt; 2min we would certainly turn interpolation off otherwise. &lt;a href=&quot;https://pfspipe.ipmu.jp/jira/secure/ViewProfile.jspa?name=rhl&quot; class=&quot;user-hover&quot; rel=&quot;rhl&quot;&gt;rhl&lt;/a&gt;?&lt;/p&gt;</comment>
                            <comment id="32531" author="rhl" created="Thu, 6 Apr 2023 15:59:35 +0000"  >&lt;p&gt;We can close this once a ticket is filed to address the problem. &#160;In the short term, a median infill of the neighbours is probably OK (and &lt;em&gt;should&lt;/em&gt; be fast). &#160;In the longer run, we need better interpolation in all arms &#8211; and this is true for Rubin too. &#160;Doug FInkbeiner claims to have fast 2-D &#160;GP interpolation (in julia&#160;&lt;img class=&quot;emoticon&quot; src=&quot;https://pfspipe.ipmu.jp/jira/images/icons/emoticons/sad.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;), but whether it&apos;d be fast enough here I don&apos;t know (the current interpolation is a simplified 1-D GP)&lt;/p&gt;</comment>
                            <comment id="32543" author="arnaud.lefur" created="Mon, 10 Apr 2023 18:31:22 +0000"  >&lt;p&gt;so I did check what scipy.ndimage has to offer, their median_filter is quick, ~3 secs, but it&apos;s not doing great with NaNs or masked array.&lt;br/&gt;
I rewrote a lousy algorithm, using np.nanmedian and np.ma.median but I couldn&apos;t reproduce their results.&lt;/p&gt;</comment>
                            <comment id="32544" author="arnaud.lefur" created="Tue, 11 Apr 2023 00:04:33 +0000"  >&lt;p&gt;I&apos;ve added a median interpolation function, just using numpy, which runs in ~30sec. &lt;br/&gt;
Results looks okay, although in my case,  the cosmic rays detection bumped from 5k to 15k.&lt;br/&gt;
5k was already too much to be real, the settings need to be tweaked in any case.&lt;/p&gt;

&lt;p&gt;I&apos;ve also noticed that the final rotation was taking ~4s, using afwMath.rotateImageBy90 but it can be reduced to few millisec by manipulating the array dimension directly.&lt;/p&gt;
</comment>
                            <comment id="32546" author="rhl" created="Wed, 12 Apr 2023 01:19:19 +0000"  >&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
ny, nx = im.shape
cube = np.zeros((9, ny - 2, nx - 2))

i = 0
cube[i] = im[0:-2, 0:-2]; i += 1
cube[i] = im[0:-2, 1:-1]; i += 1
cube[i] = im[0:-2, 2:];   i += 1

cube[i] = im[1:-1, 0:-2]; i += 1
cube[i] = im[1:-1, 1:-1]; i += 1
cube[i] = im[1:-1, 2:];   i += 1

cube[i] = im[2:, 0:-2]; i += 1
cube[i] = im[2:, 1:-1]; i += 1
cube[i] = im[2:, 2:];   i += 1

med = np.median(cube, axis=0)

im[1:-1, 1:-1] = med
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;
&lt;p&gt;I think that this perform a 3x3 median filter; it takes c. 1.1s for a 4k x 4k image.  How does this compare to your code?&lt;/p&gt;

&lt;p&gt;Also, I worry that just flipping the definition of the array rather than flipping the pixels will cost us in cache when we actually use the image for anything;  but maybe not.&lt;/p&gt;</comment>
                            <comment id="32555" author="arnaud.lefur" created="Wed, 12 Apr 2023 20:32:18 +0000"  >&lt;p&gt;I tested your code, which I modified slightly for the edge cases, but it is faster indeed, because less dimension probably, constructing a (9, 4096, 4096)  vs  (4096, 4096, 3, 3) array.&lt;/p&gt;

&lt;p&gt;Full task run now in 15s instead 30s on LAM machine.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10500" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10010" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|zzswdg:hy</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10005" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="163">PreEng11Apr1</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    </customfields>
    </item>
</channel>
</rss>