<!-- 
RSS generated by JIRA (8.3.4#803005-sha1:1f96e09b3c60279a408a2ae47be3c745f571388b) at Sat Feb 10 15:57:51 JST 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>PFS-JIRA</title>
    <link>https://pfspipe.ipmu.jp/jira</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>8.3.4</version>
        <build-number>803005</build-number>
        <build-date>13-09-2019</build-date>
    </build-info>


<item>
            <title>[PIPE2D-782] Spectra extracted from data used to construct fiberTraces are not flat</title>
                <link>https://pfspipe.ipmu.jp/jira/browse/PIPE2D-782</link>
                <project id="10002" key="PIPE2D">DRP 2-D Pipeline</project>
                    <description>&lt;p&gt;If I run the code in &lt;tt&gt;constructFiberProfilesTask.py&lt;/tt&gt; to make new fiberProfiles from a single exposure and then use them to extract the spectra from the same data they should be identically 1.0, but they are not (see  &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;13424_thumb&quot; href=&quot;https://pfspipe.ipmu.jp/jira/secure/attachment/13424/13424_three_slits.png&quot; title=&quot;three_slits.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;13424&quot; file-preview-title=&quot;three_slits.png&quot;&gt;&lt;img src=&quot;https://pfspipe.ipmu.jp/jira/secure/thumbnail/13422/_thumb_13422.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/a&gt;&lt;/span&gt; ). The code was actually unpacked into this stand-alone fragment from a notebook, but I think it&apos;s doing the same thing as the &lt;tt&gt;combine&lt;/tt&gt; method (remove the &lt;tt&gt;%matplotlib notebook&lt;/tt&gt; to run as a python script).&lt;/p&gt;
&lt;div class=&quot;code panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;codeContent panelContent&quot;&gt;
&lt;pre class=&quot;code-java&quot;&gt;
&lt;span class=&quot;code-keyword&quot;&gt;import&lt;/span&gt; os
&lt;span class=&quot;code-keyword&quot;&gt;import&lt;/span&gt; numpy as np

&lt;span class=&quot;code-keyword&quot;&gt;import&lt;/span&gt; matplotlib.pyplot as plt
%matplotlib notebook

&lt;span class=&quot;code-keyword&quot;&gt;import&lt;/span&gt; lsst.daf.persistence as dafPersist

# ---------------------------------

base, rerun = &lt;span class=&quot;code-quote&quot;&gt;&quot;/projects/HSC/PFS/Subaru&quot;&lt;/span&gt;, os.path.join(&lt;span class=&quot;code-quote&quot;&gt;&quot;rhl&quot;&lt;/span&gt;, &lt;span class=&quot;code-quote&quot;&gt;&quot;sunss&quot;&lt;/span&gt;)

dataDir = os.path.join(base, &lt;span class=&quot;code-quote&quot;&gt;&quot;rerun&quot;&lt;/span&gt;, rerun)
calibRoot = os.path.join(base, &lt;span class=&quot;code-quote&quot;&gt;&quot;CALIB-SuNSS&quot;&lt;/span&gt;)
butler = dafPersist.Butler(dataDir, calibRoot=calibRoot)

# ---------------------------------

&lt;span class=&quot;code-keyword&quot;&gt;import&lt;/span&gt; pfs.drp.stella.buildFiberProfiles 

BuildFiberProfilesTask = pfs.drp.stella.buildFiberProfiles.BuildFiberProfilesTask

config = BuildFiberProfilesTask.ConfigClass()
config.doBlindFind = False
buildFiberProfilesTask = BuildFiberProfilesTask(config)

# ---------------------------------

dataId = dict(visit=46243, spectrograph=1, arm=&lt;span class=&quot;code-quote&quot;&gt;&apos;r&apos;&lt;/span&gt;)   # domeflat

exp = butler.get(&lt;span class=&quot;code-quote&quot;&gt;&quot;calexp&quot;&lt;/span&gt;, dataId)
detMap = butler.get(&lt;span class=&quot;code-quote&quot;&gt;&quot;detectorMap&quot;&lt;/span&gt;, dataId)
pfsConfig = butler.get(&lt;span class=&quot;code-quote&quot;&gt;&quot;pfsConfig&quot;&lt;/span&gt;, dataId)

results = buildFiberProfilesTask.run(exp, detMap, pfsConfig)
profiles, centers = results.profiles, results.centers

traces = profiles.makeFiberTracesFromDetectorMap(detMap)

spectra0 = traces.extractSpectra(exp.maskedImage)

&lt;span class=&quot;code-keyword&quot;&gt;for&lt;/span&gt; i, ss in enumerate(spectra0):
    profiles[ss.fiberId].norm = ss.flux

traces = profiles.makeFiberTracesFromDetectorMap(detMap)

spectra = traces.extractSpectra(exp.maskedImage)

# ---------------------------------

fig = 1; plt.close(fig); fig = plt.figure(fig)

&lt;span class=&quot;code-keyword&quot;&gt;for&lt;/span&gt; ss in spectra:
    &lt;span class=&quot;code-keyword&quot;&gt;if&lt;/span&gt; 4 &amp;lt; ss.fiberId &amp;lt; 140:
        plt.plot(ss.flux/np.median(ss.flux), alpha=0.5)

plt.ylim(1 + 0.01*np.array([-1, 1]))
plt.xlabel(&lt;span class=&quot;code-quote&quot;&gt;&quot;pixel&quot;&lt;/span&gt;)
plt.ylabel(&lt;span class=&quot;code-quote&quot;&gt;&quot;Extracted flux&quot;&lt;/span&gt;)
plt.title(f&lt;span class=&quot;code-quote&quot;&gt;&quot;{&lt;span class=&quot;code-quote&quot;&gt;&apos;%(visit)d %(arm)s%(spectrograph)d&apos;&lt;/span&gt; % dataId}&quot;&lt;/span&gt;);
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;</description>
                <environment></environment>
        <key id="15470">PIPE2D-782</key>
            <summary>Spectra extracted from data used to construct fiberTraces are not flat</summary>
                <type id="10001" iconUrl="https://pfspipe.ipmu.jp/jira/secure/viewavatar?size=xsmall&amp;avatarId=10515&amp;avatarType=issuetype">Story</type>
                                            <priority id="10000" iconUrl="https://pfspipe.ipmu.jp/jira/images/icons/priorities/medium.svg">Normal</priority>
                        <status id="10002" iconUrl="https://pfspipe.ipmu.jp/jira/images/icons/statuses/generic.png" description="The issue is resolved, reviewed, and merged">Done</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10000">Done</resolution>
                                        <assignee username="price">price</assignee>
                                    <reporter username="rhl">rhl</reporter>
                        <labels>
                    </labels>
                <created>Tue, 16 Mar 2021 16:55:43 +0000</created>
                <updated>Fri, 19 Mar 2021 17:12:36 +0000</updated>
                            <resolved>Thu, 18 Mar 2021 19:13:40 +0000</resolved>
                                                                        <due></due>
                            <votes>0</votes>
                                    <watches>4</watches>
                                                                <comments>
                            <comment id="18867" author="price" created="Tue, 16 Mar 2021 20:05:33 +0000"  >&lt;p&gt;Note that &lt;tt&gt;profiles.makeFiberTracesFromDetectorMap(detMap)&lt;/tt&gt; doesn&apos;t use the exact same centers used in building the &lt;tt&gt;fiberProfiles&lt;/tt&gt;. For that, you want to use &lt;tt&gt;profiles.makeFiberTraces(detMap.bbox.getDimensions(), centers)&lt;/tt&gt;. But I don&apos;t think that causes what you&apos;re seeing, because you&apos;re being consistent later on.&lt;/p&gt;

&lt;p&gt;Your &lt;tt&gt;config&lt;/tt&gt; has &lt;tt&gt;profileRadius=5&lt;/tt&gt;, which is going to cause overlapping profiles in the dense fiber region (e.g., try &lt;tt&gt;profiles&lt;span class=&quot;error&quot;&gt;&amp;#91;167&amp;#93;&lt;/span&gt;.plot()&lt;/tt&gt;). Therefore, changing the normalisation has a ripple effect through all the overlaps (the off-diagonal values in the tri-di matrix are about 1/10 of the diagonal). The fibers outside the dense fiber region have values close to unity. If you iterate once (&lt;tt&gt;for ss in spectra: profiles&lt;span class=&quot;error&quot;&gt;&amp;#91;ss.fiberId&amp;#93;&lt;/span&gt;.norm *= ss.flux&lt;/tt&gt;), the resultant spectra are flat:&lt;br/&gt;
&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://pfspipe.ipmu.jp/jira/secure/attachment/13423/13423_pipe2d-782.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="18868" author="rhl" created="Tue, 16 Mar 2021 20:25:18 +0000"  >&lt;p&gt;I know about the centres difference (I think my use of the detMap is better). &#160;&lt;/p&gt;

&lt;p&gt;I thought about the overlaps, but only showed results from the non-overlapping fibres below fiberId 140 &#8212; why do problems in the dense pack part of the slit propagate out to the ends? &#160;I&apos;d expect the matrix to be blocked.&lt;/p&gt;</comment>
                            <comment id="18870" author="rhl" created="Tue, 16 Mar 2021 20:46:51 +0000"  >&lt;p&gt;Here are two more figures, showing just three fibres from the non-dense part of the slit.&#160; &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://pfspipe.ipmu.jp/jira/secure/thumbnail/13424/_thumb_13424.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;13426_thumb&quot; href=&quot;https://pfspipe.ipmu.jp/jira/secure/attachment/13426/13426_quiver-46188-fiberTrace.png&quot; title=&quot;quiver-46188-fiberTrace.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;13426&quot; file-preview-title=&quot;quiver-46188-fiberTrace.png&quot;&gt;&lt;img src=&quot;https://pfspipe.ipmu.jp/jira/secure/thumbnail/13425/_thumb_13425.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/a&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="18921" author="price" created="Thu, 18 Mar 2021 16:21:38 +0000"  >&lt;p&gt;I reproduced this with a simple setup (using the &lt;tt&gt;synthetic&lt;/tt&gt; package in drp&amp;#95;stella, which uses straight Gaussian traces). When the traces are perfectly vertical, the extractions are perfect (setting the profile normalisation to the extracted spectrum results in unity when re-extracting). When the traces are slanted, the extractions are not perfect (I see some 1% deviations when the slope is 0.02). I believe this is due to the profile not accurately modeling the data.&lt;/p&gt;</comment>
                            <comment id="18922" author="rhl" created="Thu, 18 Mar 2021 16:28:15 +0000"  >&lt;p&gt;I don&apos;t understand this, as we define the normalisation of the traces from spectra extracted using the same traces, so errors in the modelling should cancel out.&lt;/p&gt;</comment>
                            <comment id="18924" author="price" created="Thu, 18 Mar 2021 18:15:44 +0000"  >&lt;p&gt;Of course you&apos;re right, and that was the important clue that I kept forgetting: it turns out that when there was no normalisation specified, the traces did not sum to unity, but when there was a normalisation specified, then I multiplied by the normalisation divided by the sum across the trace: there was an extra normalisation.&lt;/p&gt;

&lt;p&gt;After fixing this, I&apos;m getting 5e-8 precision, even with a slope of 0.05. This will be ready for review soon.&lt;/p&gt;</comment>
                            <comment id="18925" author="price" created="Thu, 18 Mar 2021 19:06:08 +0000"  >&lt;p&gt;Demonstration that it&apos;s fixed: &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://pfspipe.ipmu.jp/jira/secure/attachment/13428/13428_pipe2d-782-fixed.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="18926" author="hassan" created="Thu, 18 Mar 2021 19:11:16 +0000"  >&lt;p&gt;Fix looks fine. No additional comments.&lt;/p&gt;</comment>
                            <comment id="18927" author="price" created="Thu, 18 Mar 2021 19:13:40 +0000"  >&lt;p&gt;Merged.&lt;/p&gt;</comment>
                            <comment id="18939" author="price" created="Fri, 19 Mar 2021 17:12:36 +0000"  >&lt;p&gt;Added another commit to move from &lt;tt&gt;profiles.makeFiberTraces&lt;/tt&gt; to &lt;tt&gt;profiles.makeFiberTracesFromDetectorMap&lt;/tt&gt;, and merged after RHL signed off on Slack.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="13425" name="extract_three_slits.png" size="168530" author="rhl" created="Tue, 16 Mar 2021 20:46:16 +0000"/>
                            <attachment id="13428" name="pipe2d-782-fixed.png" size="19895" author="price" created="Thu, 18 Mar 2021 19:05:42 +0000"/>
                            <attachment id="13423" name="pipe2d-782.png" size="19706" author="price" created="Tue, 16 Mar 2021 20:05:09 +0000"/>
                            <attachment id="13422" name="quartz_spectra.png" size="264976" author="rhl" created="Tue, 16 Mar 2021 16:50:27 +0000"/>
                            <attachment id="13424" name="three_slits.png" size="126947" author="rhl" created="Tue, 16 Mar 2021 20:46:10 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                            <customfield id="customfield_10500" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10010" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|02qpt6:508yl0chzzy0000093</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10100" key="com.atlassian.jira.plugin.system.customfieldtypes:userpicker">
                        <customfieldname>Reviewers</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>hassan</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10005" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="102">2DDRP-2021 A3</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    </customfields>
    </item>
</channel>
</rss>