BASEDIR=/work/kiyoyabe/erun/run19/reduction_new DATASTORE=/work/datastore cd $BASEDIR setup pfs_pipe2d ########### ## run19 ## ########### # PFS/calib/pipe2d-1547/run19/calibGen.20241112a ## bias ## # Use PIPE2D-1557 ## dark ## # Use PIPE2D-1557 ## detectorMaps ## # fitDetectorMap (brn) pipetask --log-level .=INFO run --register-dataset-types -j 36 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/detectorMapsGen.20241112a/brn -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (115824..115838, 115842..115844)" -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # results are bad... # bootstrap (b) pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/b2 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115740, 115743) AND arm = 'b' AND spectrograph = 2" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=10 pfs.bootstrap INFO: Matched 162 lines pfs.bootstrap INFO: Median difference from detectorMap: -6.627930,-0.971178 pixels pfs.bootstrap INFO: Fit 51/84 points, rms: x=0.053969 y=0.122313 total=0.052123 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: -10.076164,4.664388 pixels pfs.bootstrap INFO: Fit 34/78 points, rms: x=0.068233 y=0.087373 total=0.035130 pixels pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/b3 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115740, 115743) AND arm = 'b' AND spectrograph = 3" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=-15 pfs.bootstrap INFO: Matched 156 lines pfs.bootstrap INFO: Median difference from detectorMap: -2.131023,3.558056 pixels pfs.bootstrap INFO: Fit 60/73 points, rms: x=0.049307 y=0.061304 total=0.034402 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: -4.974445,3.013771 pixels pfs.bootstrap INFO: Fit 59/83 points, rms: x=0.100356 y=0.211146 total=0.114789 pixels pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/b4 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115740, 115743) AND arm = 'b' AND spectrograph = 4" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=-15 pfs.bootstrap INFO: Matched 146 lines pfs.bootstrap INFO: Median difference from detectorMap: -7.115083,0.320446 pixels pfs.bootstrap INFO: Fit 56/71 points, rms: x=0.097886 y=0.136950 total=0.071179 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: -9.820977,1.354247 pixels pfs.bootstrap INFO: Fit 56/75 points, rms: x=0.075430 y=0.102706 total=0.057269 pixels # bootstrap (r) pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/r2 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115737, 115743) AND arm = 'r' AND spectrograph = 2" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=0 -c bootstrap:spatialOffset=9 pfs.bootstrap INFO: Matched 689 lines pfs.bootstrap INFO: Median difference from detectorMap: 3.500673,-0.472853 pixels pfs.bootstrap INFO: Fit 275/349 points, rms: x=0.126303 y=0.217430 total=0.108355 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: -2.789952,0.999808 pixels pfs.bootstrap INFO: Fit 308/340 points, rms: x=0.112214 y=0.390307 total=0.217354 pixels pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/r3 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115737, 115743) AND arm = 'r' AND spectrograph = 3" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=20 pfs.bootstrap INFO: Matched 641 lines pfs.bootstrap INFO: Median difference from detectorMap: -0.160638,-1.702519 pixels pfs.bootstrap INFO: Fit 246/324 points, rms: x=0.147693 y=0.098490 total=0.075642 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: -1.982070,-1.292657 pixels pfs.bootstrap INFO: Fit 224/317 points, rms: x=0.175839 y=0.291888 total=0.146682 pixels pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/r4 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115737, 115743) AND arm = 'r' AND spectrograph = 4" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=-15 pfs.bootstrap INFO: Matched 698 lines pfs.bootstrap INFO: Median difference from detectorMap: 10.641655,3.648760 pixels pfs.bootstrap INFO: Fit 265/352 points, rms: x=0.089076 y=0.247063 total=0.128688 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: 6.255600,1.073054 pixels pfs.bootstrap INFO: Fit 242/346 points, rms: x=0.122057 y=0.186041 total=0.089377 pixels # bootstrap (n) pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/n2 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115737, 115743) AND arm = 'n' AND spectrograph = 2" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=15 pfs.bootstrap INFO: Matched 268 lines pfs.bootstrap INFO: Median difference from detectorMap: 0.148891,-3.018109 pixels pfs.bootstrap INFO: Fit 227/268 points, rms: x=0.061957 y=0.227223 total=0.133486 pixels pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/n3 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115737, 115743) AND arm = 'n' AND spectrograph = 3" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=0 pfs.bootstrap INFO: Matched 256 lines pfs.bootstrap INFO: Median difference from detectorMap: 6.266899,-2.111827 pixels pfs.bootstrap INFO: Fit 222/256 points, rms: x=0.081036 y=0.411498 total=0.267891 pixels pipetask --log-level .=INFO run --register-dataset-types -j 20 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/n4 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115737, 115743) AND arm = 'n' AND spectrograph = 4" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=0 pfs.bootstrap INFO: Matched 305 lines pfs.bootstrap INFO: Median difference from detectorMap: -13.245268,-4.777100 pixels pfs.bootstrap INFO: Fit 209/305 points, rms: x=0.109683 y=0.151423 total=0.087588 pixels # bootstrap (m) pipetask --log-level .=INFO run --register-dataset-types -j 18 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/m2 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115746, 115749) AND arm = 'm' AND spectrograph = 2" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spatialOffset=-8 -c bootstrap:spectralOffset=-5 pfs.bootstrap INFO: Matched 292 lines pfs.bootstrap INFO: Median difference from detectorMap: 3.069882,-1.769149 pixels pfs.bootstrap INFO: Fit 129/154 points, rms: x=0.106261 y=0.245380 total=0.126307 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: -3.257051,-0.690398 pixels pfs.bootstrap INFO: Fit 114/138 points, rms: x=0.063369 y=0.325677 total=0.174102 pixels pipetask --log-level .=INFO run --register-dataset-types -j 18 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/m3 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115746, 115749) AND arm = 'm' AND spectrograph = 3" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spatialOffset=-8 -c bootstrap:spectralOffset=9 pfs.bootstrap INFO: Matched 287 lines pfs.bootstrap INFO: Median difference from detectorMap: 0.623646,-0.423908 pixels pfs.bootstrap INFO: Fit 107/144 points, rms: x=0.141108 y=0.065200 total=0.072863 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: -1.482604,0.140654 pixels pfs.bootstrap INFO: Fit 104/143 points, rms: x=0.167697 y=0.359896 total=0.195695 pixels pipetask --log-level .=INFO run --register-dataset-types -j 18 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run19/bootstrap/m4 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (115746, 115749) AND arm = 'm' AND spectrograph = 4" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spatialOffset=2 -c bootstrap:spectralOffset=-20 pfs.bootstrap INFO: Matched 382 lines pfs.bootstrap INFO: Median difference from detectorMap: 1.848628,1.860507 pixels pfs.bootstrap INFO: Fit 150/193 points, rms: x=0.099435 y=0.476130 total=0.250895 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: -2.610057,-1.880794 pixels pfs.bootstrap INFO: Fit 146/189 points, rms: x=0.140644 y=0.450197 total=0.197041 pixels butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/b2 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/b3 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/b4 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/r2 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/r3 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/r4 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/n2 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/n3 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/n4 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/m2 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/m3 PFS/bootstrap/run19 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/bootstrap/m4 PFS/bootstrap/run19 detectorMap_bootstrap butlerCleanRun.py $DATASTORE PFS/calib/pipe2d-1547/run19/bootstrap/* postISRCCD pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run19,PFS/defaults -o PFS/calib/pipe2d-1547/run19/detectorMapsGen.20241112a/brn -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (115824..115838, 115842..115844)" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run19,PFS/defaults -o PFS/calib/pipe2d-1547/run19/detectorMapsGen.20241112a/m -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (115957..115958, 115961..115972) AND arm = 'm'" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast (b2) xRMS=0.035348 yRMS=0.056018 xSoften=0.030727 ySoften=0.056720 (b3) xRMS=0.030906 yRMS=0.056381 xSoften=0.024389 ySoften=0.054303 (b4) xRMS=0.030744 yRMS=0.041485 xSoften=0.024458 ySoften=0.032816 (r2) xRMS=0.027173 yRMS=0.042126 xSoften=0.021879 ySoften=0.027618 (r3) xRMS=0.028683 yRMS=0.041771 xSoften=0.023176 ySoften=0.028074 (r4) xRMS=0.027900 yRMS=0.041295 xSoften=0.022455 ySoften=0.026551 (n2) xRMS=0.041232 yRMS=0.069443 xSoften=0.039853 ySoften=0.056766 (n3) xRMS=0.038740 yRMS=0.071283 xSoften=0.036586 ySoften=0.060179 (n4) xRMS=0.031798 yRMS=0.073881 xSoften=0.029249 ySoften=0.067921 (m2) xRMS=0.032828 yRMS=0.038522 xSoften=0.028940 ySoften=0.025913 (m3) xRMS=0.031735 yRMS=0.040748 xSoften=0.026972 ySoften=0.028773 (m4) xRMS=0.030671 yRMS=0.036523 xSoften=0.026181 ySoften=0.022964 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run19/detectorMapsGen.20241112a/brn PFS/calib/pipe2d-1547/run19/calibGen.20241112a --instrument PFS --begin-date 2024-10-16T00:00:00 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run19/detectorMapsGen.20241112a/m PFS/calib/pipe2d-1547/run19/calibGen.20241112a --instrument PFS --begin-date 2024-10-16T00:00:00 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run19/detectorMapsGen.20241112a/* postISRCCD ## fiberProfiles ## ## brn ## # defineFiberProfilesInputs.py $DATASTORE PFS run19_brn --bright 116168..116176 --dark 116140..116148 --bright 116234..116242 --dark 116206..116214 --bright 116386..116394 --dark 116358..116366 --bright 116584..116592 --dark 116556..116564 pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run19/fiberProfilesGen.20241112a/brn -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run19_brn'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/fiberProfilesGen.20241112a/brn PFS/calib/pipe2d-1547/run19/calibGen.20241112a fiberProfiles --begin-date 2024-10-16T00:00:00 ## m ## # defineFiberProfilesInputs.py --update /work/datastore PFS run19_m --bright 116159..116167 --dark 116149..116157 --bright 116225..116233 --dark 116215..116223 --bright 116377..116385 --dark 116367..116375 --bright 116575..116583 --dark 116565..116573 pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run19/fiberProfilesGen.20241112a/m -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run19_m' and arm = 'm'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run19/fiberProfilesGen.20241112a/m PFS/calib/pipe2d-1547/run19/calibGen.20241112a fiberProfiles --begin-date 2024-10-16T00:00:00 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run19/fiberProfilesGen.20241112a/* postISRCCD ## fiberNorms ## source ./generate_fiberNorms_pipe2d-1547_run19_20241112.sh ## verification ## pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run19/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (115893, 115894)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run19/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (116089, 116090)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run19/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (116095, 116096)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run19/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (116098, 116099)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run19/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (116081..116084)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run19/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (115938)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run19/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (115913)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 ## dmQa ## pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run19/verify.20241112a -o PFS/calib/pipe2d-1547/run19/dmQa.20241112a -p '$DRP_QA_DIR/pipelines/drpQA.yaml#dmResiduals' -d "visit IN (116081..116084)" --fail-fast -j 32 ########### ## run18 ## ########### # PFS/calib/pipe2d-1547/run18/calibGen.20241112a ## bias ## # Use PIPE2D-1557 ## dark ## # Use PIPE2D-1557 ## ## detectorMaps (m2,m3,m4) ## ==> note that we need run19 bootstrap # 113981..113995 or 114986..114990 pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run19,PFS/defaults -o PFS/calib/pipe2d-1547/run18/detectorMapsGen.20241112a/m -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (113981..113995) AND arm = 'm' AND spectrograph IN (2, 3, 4)" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast ## detectorMaps (m1) ## ==> note that we need run17 bootstrap pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run17,PFS/defaults -o PFS/calib/pipe2d-1547/run18/detectorMapsGen.20241112a/m1 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (113981..113995) AND arm = 'm' AND spectrograph = 1" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast (m1) xRMS=0.029460 yRMS=0.039658 xSoften=0.025484 ySoften=0.027860 (m2) xRMS=0.030345 yRMS=0.039141 xSoften=0.026357 ySoften=0.026234 (m3) xRMS=0.030172 yRMS=0.056327 xSoften=0.025258 ySoften=0.048656 (m4) xRMS=0.028283 yRMS=0.038789 xSoften=0.023576 ySoften=0.021242 butlerDecertify.py $DATASTORE PFS/calib/pipe2d-1547/run18/calibGen.20241112a detectorMap_calib --begin-date 2024-08-24T00:00:00 --end-date 2024-10-15T23:59:59.9999 --id instrument=PFS spectrograph=1 arm=m butlerDecertify.py $DATASTORE PFS/calib/pipe2d-1547/run18/calibGen.20241112a detectorMap_calib --begin-date 2024-08-24T00:00:00 --end-date 2024-10-15T23:59:59.9999 --id instrument=PFS spectrograph=2 arm=m butlerDecertify.py $DATASTORE PFS/calib/pipe2d-1547/run18/calibGen.20241112a detectorMap_calib --begin-date 2024-08-24T00:00:00 --end-date 2024-10-15T23:59:59.9999 --id instrument=PFS spectrograph=3 arm=m butlerDecertify.py $DATASTORE PFS/calib/pipe2d-1547/run18/calibGen.20241112a detectorMap_calib --begin-date 2024-08-24T00:00:00 --end-date 2024-10-15T23:59:59.9999 --id instrument=PFS spectrograph=4 arm=m certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run18/detectorMapsGen.20241112a/m/20241114T201506Z PFS/calib/pipe2d-1547/run18/calibGen.20241112a --instrument PFS --begin-date 2024-08-24T00:00:00 --end-date 2024-10-15T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run18/detectorMapsGen.20241112a/m1/20241114T203832Z PFS/calib/pipe2d-1547/run18/calibGen.20241112a --instrument PFS --begin-date 2024-08-24T00:00:00 --end-date 2024-10-15T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run18/detectorMapsGen.20241112a/* postISRCCD ## fiberProfiles ## ## m ## defineFiberProfilesInputs.py --update /work/datastore PFS run18_m --bright 114294..114302 --dark 114284..114292 --bright 114342..114350 --dark 114332..114340 --bright 114390..114398 --dark 114380..114388 --bright 114477..114485 --dark 114467..114475 pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/calibGen.20241112a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run18/fiberProfilesGen.20241112a/m -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run18_m' and arm = 'm'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run18/fiberProfilesGen.20241112a/m PFS/calib/pipe2d-1547/run18/calibGen.20241112a fiberProfiles --begin-date 2024-08-24T00:00:00 --end-date 2024-10-15T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run18/fiberProfilesGen.20241112a/* postISRCCD ## fiberNorms ## source ./generate_fiberNorms_pipe2d-1547_run18_20241112.sh ## verification (some science data) ## pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run18/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (115073, 115074)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run18/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (115028, 115029)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run18/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (115066, 115067)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run18/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (115021, 115022)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run18/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/calibrateExposure.yaml' -d "visit IN (115049, 115050)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 ## verification (some quartz data) ## pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run18/verify.20241112a -p '$DRP_STELLA_DIR/pipelines/observing.yaml' -d "visit IN (115017, 115023, 115024, 115026, 115026, 115030, 115031, 115033, 115033, 115037, 115038, 115040, 115047, 115051, 115052, 115054, 115061, 115062, 115064, 115068, 115069, 115071, 115075, 115076, 115078)" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run18/verify.20241112a/* postISRCCD ## verification (tune params ) ## #pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run18/verify.20241112b -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (115028, 115029)" --fail-fast -j 32 -c mergeArms:fitSkyModel.blockSize=50 -c mergeArms:fitSkyModel.oversample=1.5 -c reduceExposure:doApplyFiberNorms=False ## dmQa ## pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run18/verify.20241112a -o PFS/calib/pipe2d-1547/run18/dmQa.20241112a -p '$DRP_QA_DIR/pipelines/drpQA.yaml#dmResiduals' -d "visit IN (115066, 115067)" --fail-fast -j 32 ########### ## run17 ## ########### # PFS/calib/pipe2d-1547/run17/calibGen.20241122a ## bias ## pipetask run --register-dataset-types -j 32 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/raw/sps,PFS/calib -o PFS/calib/pipe2d-1547/run16/biasGen.20241112a -p $DRP_STELLA_DIR/pipelines/bias.yaml -d "visit IN (109212..109236)" --fail-fast butler certify-calibrations $DATASTORE PFS/calib/pipe2d-1547/run16/biasGen.20241112a PFS/calib/pipe2d-1547/run16/calibGen.20241112a bias --begin-date 2024-05-01T00:00:00 --end-date 2024-08-23T23:59:59.9999 butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run16/calibGen.20241112a bias butler certify-calibrations $DATASTORE PFS/calib/pipe2d-1547/run16/biasGen.20241112a PFS/calib/pipe2d-1547/run16/calibGen.20241112a bias --begin-date 2024-04-15T00:00:00 --end-date 2024-08-23T23:59:59.9999 #butler certify-calibrations $DATASTORE PFS/calib/pipe2d-1547/run16/biasGen.20241112a PFS/calib/pipe2d-1547/run17/calibGen.20241122a bias --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 butler certify-calibrations $DATASTORE PFS/calib/pipe2d-1547/run16/biasGen.20241112a PFS/calib/pipe2d-1547/run17/calibGen.20241122b bias --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run16/biasGen.20241112a/* biasProc ## dark ## pipetask run --register-dataset-types -j 32 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run16/darkGen.20241112a -p $DRP_STELLA_DIR/pipelines/dark.yaml -d "visit IN (109237..109261)" --fail-fast butler certify-calibrations $DATASTORE PFS/calib/pipe2d-1547/run16/darkGen.20241112a PFS/calib/pipe2d-1547/run16/calibGen.20241112a dark --begin-date 2024-05-01T00:00:00 --end-date 2024-08-23T23:59:59.9999 butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run16/calibGen.20241112a dark butler certify-calibrations $DATASTORE PFS/calib/pipe2d-1547/run16/darkGen.20241112a PFS/calib/pipe2d-1547/run16/calibGen.20241112a dark --begin-date 2024-04-15T00:00:00 --end-date 2024-08-23T23:59:59.9999 #butler certify-calibrations $DATASTORE PFS/calib/pipe2d-1547/run16/darkGen.20241112a PFS/calib/pipe2d-1547/run17/calibGen.20241122a dark --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 butler certify-calibrations $DATASTORE PFS/calib/pipe2d-1547/run16/darkGen.20241112a PFS/calib/pipe2d-1547/run17/calibGen.20241122b dark --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 butlerCleanRun.py $DATASTORE PFS/calib/pipe2d-1547/run16/darkGen.20241112a/* darkProc darkScaled ## bootstrap (run17) for m-band ## # 111236: scienceTrace # 111254: scienceArc pipetask --log-level .=INFO run --register-dataset-types -j 18 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run17/bootstrap/m1 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (111254, 111236) AND arm = 'm' AND spectrograph = 1" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spatialOffset=0 -c bootstrap:spectralOffset=-18 pfs.bootstrap INFO: Matched 407 lines pfs.bootstrap INFO: Median difference from detectorMap: 2.227985,-0.702459 pixels pfs.bootstrap INFO: Fit 166/205 points, rms: x=0.052378 y=0.148719 total=0.085345 pixels pfs.bootstrap INFO: Updating detectorMap... pfs.bootstrap INFO: Median difference from detectorMap: 0.510403,0.754261 pixels pfs.bootstrap INFO: Fit 156/202 points, rms: x=0.084154 y=0.117485 total=0.070803 pixels butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run17/bootstrap/m1 PFS/bootstrap/run17 detectorMap_bootstrap butlerCleanRun.py $DATASTORE PFS/calib/pipe2d-1547/run17/bootstrap/* postISRCCD ## detector map ## ## generate run18 detector map ## # brn pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/brn-run18 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (114641..114661)" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # m1 pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run17,PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/m1-run18 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (113981..113995) AND arm = 'm' AND spectrograph = 1" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # m2,m3,m4 pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run19,PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/m-run18 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (113981..113995) AND arm = 'm' AND spectrograph IN (2, 3, 4)" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast #butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run17/tmpCalib.20241122a detectorMap_calib certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/brn-run18/20241125T000513Z PFS/calib/pipe2d-1547/run17/tmpCalib.20241122a --instrument PFS --end-date 2024-08-23T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/m1-run18/20241125T002916Z PFS/calib/pipe2d-1547/run17/tmpCalib.20241122a --instrument PFS --end-date 2024-08-23T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/m-run18/20241125T004808Z PFS/calib/pipe2d-1547/run17/tmpCalib.20241122a --instrument PFS --end-date 2024-08-23T23:59:59.9999 ## generate run17 detector map ## # brn #pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/tmpCalib.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/brn -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (110921..110926, 110928..110942)" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/tmpCalib.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122b/brn -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (110921..110926, 110928..110942)" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # m #pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/tmpCalib.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/m -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (110946..110963) AND arm = 'm'" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/tmpCalib.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122b/m -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (110946..110963) AND arm = 'm'" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # certify #certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/brn PFS/calib/pipe2d-1547/run17/calibGen.20241122a --instrument PFS --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 #certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/m PFS/calib/pipe2d-1547/run17/calibGen.20241122a --instrument PFS --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122b/brn PFS/calib/pipe2d-1547/run17/calibGen.20241122b --instrument PFS --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122b/m PFS/calib/pipe2d-1547/run17/calibGen.20241122b --instrument PFS --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 # n2 #pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/n2 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (110921..110926, 110928..110942) AND arm = 'n' AND spectrograph = 2" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122b,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122b/n2 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (110921..110926, 110928..110942) AND arm = 'n' AND spectrograph = 2" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast #butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run17/calibGen.20241122a detectorMap_calib --id instrument=PFS spectrograph=2 arm=n #certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/n2 PFS/calib/pipe2d-1547/run17/calibGen.20241122a --instrument PFS --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run17/calibGen.20241122b detectorMap_calib --id instrument=PFS spectrograph=2 arm=n certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122b/n2 PFS/calib/pipe2d-1547/run17/calibGen.20241122b --instrument PFS --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 #butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/* postISRCCD butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122b/* postISRCCD (b1) xRMS=0.031242 yRMS=0.075171 xSoften=0.025419 ySoften=0.079290 (b2) xRMS=0.035751 yRMS=0.061123 xSoften=0.031005 ySoften=0.060578 (b3) xRMS=0.030511 yRMS=0.054695 xSoften=0.023548 ySoften=0.052521 (b4) xRMS=0.070481 yRMS=0.047994 xSoften=0.081014 ySoften=0.040215 (r1) xRMS=0.026517 yRMS=0.043753 xSoften=0.022097 ySoften=0.031608 (r2) xRMS=0.025283 yRMS=0.041856 xSoften=0.020460 ySoften=0.027861 (r3) xRMS=0.026992 yRMS=0.044552 xSoften=0.021640 ySoften=0.033758 (r4) xRMS=0.029396 yRMS=0.040939 xSoften=0.024287 ySoften=0.027811 (n2) xRMS=0.041322 yRMS=0.079367 xSoften=0.040094 ySoften=0.069273 (n3) xRMS=0.037358 yRMS=0.079828 xSoften=0.035301 ySoften=0.069446 (m1) xRMS=0.028253 yRMS=0.040255 xSoften=0.024586 ySoften=0.028797 (m2) xRMS=0.033382 yRMS=0.040827 xSoften=0.030397 ySoften=0.029004 (m3) xRMS=0.028306 yRMS=0.048806 xSoften=0.023770 ySoften=0.040202 (m4) xRMS=0.028137 yRMS=0.037469 xSoften=0.023072 ySoften=0.019262 ## fiberProfiles ## ## brn ## defineFiberProfilesInputs.py $DATASTORE PFS run17_brn --bright 111227..111235 --dark 111217..111225 --bright 111980^111981^111984..111990 --dark 111970..111978 --bright 112031..112039 --dark 112020..112026^112028^112029 --bright 112080..112088 --dark 112069..112075^112077^112078 --update #pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run17/fiberProfilesGen.20241122a/brn -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run17_brn'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122b,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run17/fiberProfilesGen.20241122b/brn -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run17_brn'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 #butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run17/fiberProfilesGen.20241122a/brn PFS/calib/pipe2d-1547/run17/calibGen.20241122a fiberProfiles --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run17/fiberProfilesGen.20241122b/brn PFS/calib/pipe2d-1547/run17/calibGen.20241122b fiberProfiles --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 #butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run17/fiberProfilesGen.20241122a/* postISRCCD butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run17/fiberProfilesGen.20241122b/* postISRCCD ## m ==> use run18 profiles ## butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run18/fiberProfilesGen.20241112a/m PFS/calib/pipe2d-1547/run17/calibGen.20241122b fiberProfiles --begin-date 2024-05-28T00:00:00 --end-date 2024-08-23T23:59:59.9999 ## fiberNorms ## source ./generate_fiberNorms_pipe2d-1547_run17_20241122.sh ## verification ## defineVisitGroup.py /work/datastore PFS 111803 111804 111805 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122b,PFS/defaults -o PFS/calib/pipe2d-1547/run17/verify.20241122b -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (111803..111805) " --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 defineVisitGroup.py /work/datastore PFS 111009 111010 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122b,PFS/defaults -o PFS/calib/pipe2d-1547/run17/verify.20241122b -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (111009..111010) " --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 defineVisitGroup.py /work/datastore PFS 111316 111317 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122b,PFS/defaults -o PFS/calib/pipe2d-1547/run17/verify.20241122b -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (111316..111317) " --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 ########### ## run16 ## ########### # PFS/calib/pipe2d-1547/run16/calibGen.20241112a ## bias & dark (see run17 bias & dark) ## ## bootstrap ## ## detector map ## # hack run17 detectorMaps # note that this way is not a good idea actually and # should regenerate the run17 detectorMaps as a different RUN collection and certify butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run17/calibGen.20241122a detectorMap_calib certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/brn/20241125T015137Z PFS/calib/pipe2d-1547/run17/calibGen.20241122a --instrument PFS --begin-date 2024-05-01T00:00:00 --end-date 2024-08-23T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/m/20241125T023051Z PFS/calib/pipe2d-1547/run17/calibGen.20241122a --instrument PFS --begin-date 2024-05-01T00:00:00 --end-date 2024-08-23T23:59:59.9999 butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run17/calibGen.20241122a detectorMap_calib --id instrument=PFS spectrograph=2 arm=n certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run17/detectorMapsGen.20241122a/n2/20241125T034829Z PFS/calib/pipe2d-1547/run17/calibGen.20241122a --instrument PFS --begin-date 2024-05-01T00:00:00 --end-date 2024-08-23T23:59:59.9999 # brn pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run16/detectorMapsGen.20241122a/brn -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (109290..109304, 109306..109308)" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # m pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run16/detectorMapsGen.20241122a/m -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (109402..109416) AND arm = 'm'" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # certify certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run16/detectorMapsGen.20241122a/brn PFS/calib/pipe2d-1547/run16/calibGen.20241112a --instrument PFS --begin-date 2024-05-01T00:00:00 --end-date 2024-05-27T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run16/detectorMapsGen.20241122a/m PFS/calib/pipe2d-1547/run16/calibGen.20241112a --instrument PFS --begin-date 2024-05-01T00:00:00 --end-date 2024-05-27T23:59:59.9999 ## fix n2 detectorMaps (using run15 bootstrap) ## pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run15,PFS/calib/pipe2d-1547/run17/calibGen.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run16/detectorMapsGen.20241122a/n2 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (109290..109304, 109306..109308) AND spectrograph = 2 AND arm = 'n'" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run16/calibGen.20241112a detectorMap_calib --id instrument=PFS spectrograph=2 arm=n certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run16/detectorMapsGen.20241122a/n2 PFS/calib/pipe2d-1547/run16/calibGen.20241112a --instrument PFS --begin-date 2024-05-01T00:00:00 --end-date 2024-05-27T23:59:59.9999 (b1) xRMS=0.032555 yRMS=0.044571 xSoften=0.027055 ySoften=0.038835 (b2) xRMS=0.033468 yRMS=0.043772 xSoften=0.028244 ySoften=0.039398 (b3) xRMS=0.031115 yRMS=0.057296 xSoften=0.024664 ySoften=0.056455 (b4) xRMS=0.030871 yRMS=0.045794 xSoften=0.024359 ySoften=0.041560 (r1) xRMS=0.028704 yRMS=0.041979 xSoften=0.023885 ySoften=0.026798 (r2) xRMS=0.026053 yRMS=0.040775 xSoften=0.020277 ySoften=0.021578 (r3) xRMS=0.028996 yRMS=0.042909 xSoften=0.023484 ySoften=0.024732 (r4) xRMS=0.028525 yRMS=0.040990 xSoften=0.023014 ySoften=0.024053 (n2) xRMS=0.043229 yRMS=0.069471 xSoften=0.042024 ySoften=0.057572 (n3) xRMS=0.037875 yRMS=0.071492 xSoften=0.035700 ySoften=0.061145 (m1) xRMS=0.029285 yRMS=0.038093 xSoften=0.025406 ySoften=0.027879 (m2) xRMS=0.027836 yRMS=0.036069 xSoften=0.023371 ySoften=0.024092 (m3) xRMS=0.030310 yRMS=0.042235 xSoften=0.025698 ySoften=0.032107 (m4) xRMS=0.029315 yRMS=0.037677 xSoften=0.024852 ySoften=0.026192 ## fiberProfiles ## # brn defineFiberProfilesInputs.py $DATASTORE PFS run16_brn --bright 109668..109676 --dark 109652..109658 --bright 109730..109738 --dark 109722..109728 --bright 109783..109791 --dark 109770..109776 --bright 109831..109833^109835..109841 --dark 109823..109829 --update pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run16/fiberProfilesGen.20241122a/brn -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run16_brn'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run16/fiberProfilesGen.20241122a/brn PFS/calib/pipe2d-1547/run16/calibGen.20241112a fiberProfiles --begin-date 2024-05-01T00:00:00 --end-date 2024-05-27T23:59:59.9999 # m defineFiberProfilesInputs.py $DATASTORE PFS run16_m --bright 109935..109943 --dark 109927..109933 --bright 110016..110024 --dark 110008..110014 --bright 110095..110103 --dark 110087..110093 --bright 110168..110176 --dark 110160..110166 --update pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run16/fiberProfilesGen.20241122a/m -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run16_m' AND arm = 'm'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run16/fiberProfilesGen.20241122a/m PFS/calib/pipe2d-1547/run16/calibGen.20241112a fiberProfiles --begin-date 2024-05-01T00:00:00 --end-date 2024-05-27T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run16/fiberProfilesGen.20241122a/* postISRCCD # n2 fiberProfiles (after detectorMap fix) pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run16/fiberProfilesGen.20241122a/n2 -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run16_brn' AND spectrograph = 2 AND arm = 'n'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run16/calibGen.20241112a fiberProfiles --id instrument=PFS spectrograph=2 arm=n butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run16/fiberProfilesGen.20241122a/n2 PFS/calib/pipe2d-1547/run16/calibGen.20241112a fiberProfiles --begin-date 2024-05-01T00:00:00 --end-date 2024-05-27T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run16/fiberProfilesGen.20241122a/* postISRCCD ## fiberNorms ## source ./generate_fiberNorms_pipe2d-1547_run16_20241122.sh ## verification ## #defineVisitGroup.py /work/datastore PFS 110627 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run16/verify.20241122a -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (110627) " --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 #defineVisitGroup.py /work/datastore PFS 110604 110605 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run16/verify.20241122a -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (110604..110605) " --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 ########### ## run15 ## ########### # PFS/calib/pipe2d-1547/run15/calibGen.20241125a ## bias ## pipetask run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/raw/sps,PFS/calib -o PFS/calib/pipe2d-1547/run15/biasGen.20241125a -p $DRP_STELLA_DIR/pipelines/bias.yaml -d "visit IN (106544..106564)" --fail-fast butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/biasGen.20241125a PFS/calib/pipe2d-1547/run15/calibGen.20241125a bias --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run15/biasGen.20241125a/* biasProc ## dark ## pipetask run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/darkGen.20241125a -p $DRP_STELLA_DIR/pipelines/dark.yaml -d "visit IN (106497..106503) AND arm IN ('b', 'r')" --fail-fast butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/darkGen.20241125a PFS/calib/pipe2d-1547/run15/calibGen.20241125a dark --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 pipetask run --register-dataset-types -j 32 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/darkGen.20241125a.n -p $DRP_STELLA_DIR/pipelines/dark.yaml -d "visit IN (109237..109261) AND arm = 'n'" --fail-fast butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/darkGen.20241125a.n PFS/calib/pipe2d-1547/run15/calibGen.20241125a dark --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run15/darkGen.20241125a/* darkProc butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run15/darkGen.20241125a.n/* darkProc ## bootstrap for SM2 ## # b2 pipetask --log-level .=INFO run --register-dataset-types -j 8 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/bootstrap/b2 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (107126, 107578) AND arm = 'b' AND spectrograph = 2" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spatialOffset=-4 -c bootstrap:spectralOffset=10 -c bootstrap:findLines.threshold=25.0 # r2 pipetask --log-level .=INFO run --register-dataset-types -j 8 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/bootstrap/r2 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (107126, 107578) AND arm = 'r' AND spectrograph = 2" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spectralOffset=10 # n2 pipetask --log-level .=INFO run --register-dataset-types -j 8 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/bootstrap/n2 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (107126, 107578) AND arm = 'n' AND spectrograph = 2" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spatialOffset=-3 -c bootstrap:spectralOffset=11 -c bootstrap:findLines.threshold=25.0 # m2 pipetask --log-level .=INFO run --register-dataset-types -j 8 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/bootstrap/m2 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (107127, 107576) AND arm = 'm' AND spectrograph = 2" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c 'bootstrap:targetType=[ENGINEERING]' -c bootstrap:spatialOffset=0 -c bootstrap:spectralOffset=10 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/bootstrap/b2 PFS/bootstrap/run15 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/bootstrap/r2 PFS/bootstrap/run15 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/bootstrap/n2 PFS/bootstrap/run15 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/bootstrap/m2 PFS/bootstrap/run15 detectorMap_bootstrap butlerDecertify.py /work/datastore PFS/bootstrap/run15 detectorMap_bootstrap --id instrument=PFS spectrograph=2 arm=b butlerDecertify.py /work/datastore PFS/bootstrap/run15 detectorMap_bootstrap --id instrument=PFS spectrograph=2 arm=n butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/bootstrap/b2 PFS/bootstrap/run15 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/bootstrap/n2 PFS/bootstrap/run15 detectorMap_bootstrap ## detectorMap for SM2 ## pipetask --log-level .=INFO run --register-dataset-types -j 64 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run15,PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/brn2 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (106838..106870) AND spectrograph = 2" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run15,PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/m2 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (106871..106902) AND spectrograph = 2 AND arm = 'm'" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast #pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run15,PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/n2 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (106838..106870) AND spectrograph = 2 AND arm = 'n'" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast ## detectorMap for other SMs (based on run16 detectorMaps) ## pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/brn-run16 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (109290..109304, 109306..109308) AND spectrograph IN (1, 3, 4)" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run17/calibGen.20241122a,PFS/calib/pipe2d-1547/run16/calibGen.20241112a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/m-run16 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (109402..109416) AND arm = 'm' AND spectrograph IN (1, 3, 4)" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/brn-run16 PFS/calib/pipe2d-1547/run15/calibTmp.20241125a --instrument PFS certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/m-run16 PFS/calib/pipe2d-1547/run15/calibTmp.20241125a --instrument PFS # brn pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibTmp.20241125a,PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/brn134 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (106838..106870) AND spectrograph IN (1, 3, 4)" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # m pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibTmp.20241125a,PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/m134 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (106871..106902) AND spectrograph IN (1, 3, 4) AND arm = 'm'" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/brn134 PFS/calib/pipe2d-1547/run15/calibGen.20241125a --instrument PFS --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/m134 PFS/calib/pipe2d-1547/run15/calibGen.20241125a --instrument PFS --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/brn2/20241126T223828Z PFS/calib/pipe2d-1547/run15/calibGen.20241125a --instrument PFS --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run15/detectorMapsGen.20241125a/m2 PFS/calib/pipe2d-1547/run15/calibGen.20241125a --instrument PFS --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 (b1) xRMS=0.031696 yRMS=0.045586 xSoften=0.026070 ySoften=0.041968 (b2) xRMS=0.035521 yRMS=0.056478 xSoften=0.029836 ySoften=0.060207 (b4) xRMS=0.030330 yRMS=0.046668 xSoften=0.023992 ySoften=0.043811 (r1) xRMS=0.029868 yRMS=0.039383 xSoften=0.025067 ySoften=0.023004 (r2) xRMS=0.029756 yRMS=0.039053 xSoften=0.024012 ySoften=0.027357 (r3) xRMS=0.031049 yRMS=0.036659 xSoften=0.025915 ySoften=0.023651 (r4) xRMS=0.031005 yRMS=0.039617 xSoften=0.026009 ySoften=0.021448 (n2) xRMS=0.042402 yRMS=0.058302 xSoften=0.040784 ySoften=0.042962 (n3) xRMS=0.036850 yRMS=0.060881 xSoften=0.034820 ySoften=0.044831 (m1) xRMS=0.031169 yRMS=0.034772 xSoften=0.027090 ySoften=0.025461 (m2) xRMS=0.031380 yRMS=0.036271 xSoften=0.026039 ySoften=0.026628 (m3) xRMS=0.031102 yRMS=0.043761 xSoften=0.026389 ySoften=0.036223 (m4) xRMS=0.029989 yRMS=0.032494 xSoften=0.025135 ySoften=0.021936 ## fiberProfiles ## # brn defineFiberProfilesInputs.py $DATASTORE PFS run15_brn --bright 106956..106964 --dark 106948..106954 --bright 107002..107010 --dark 106994..107000 --bright 107048..107056 --dark 107040..107046 --bright 107096..107104 --dark 107088..107094 --update pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run15/fiberProfilesGen.20241125a/brn -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run15_brn'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/fiberProfilesGen.20241125a/brn PFS/calib/pipe2d-1547/run15/calibGen.20241125a fiberProfiles --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 # m defineFiberProfilesInputs.py $DATASTORE PFS run15_m --bright 107218..107226 --dark 107210..107216 --bright 107264..107272 --dark 107256..107262 --bright 107372..107380 --dark 107364..107370 --bright 107537..107547 --dark 107527..107535 --update pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run15/fiberProfilesGen.20241125a/m -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run15_m' AND arm = 'm'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/fiberProfilesGen.20241125a/m PFS/calib/pipe2d-1547/run15/calibGen.20241125a fiberProfiles --begin-date 2024-03-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 # change the validity range of fiberProfiles butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run15/calibGen.20241125a fiberProfiles butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/fiberProfilesGen.20241125a/brn PFS/calib/pipe2d-1547/run15/calibGen.20241125a fiberProfiles --begin-date 2000-01-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run15/fiberProfilesGen.20241125a/m PFS/calib/pipe2d-1547/run15/calibGen.20241125a fiberProfiles --begin-date 2000-01-01T00:00:00 --end-date 2024-04-30T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run15/fiberProfilesGen.20241125a/* postISRCCD ## fiberNorms ## source ./generate_fiberNorms_pipe2d-1547_run15_20241125.sh ## verification ## #defineVisitGroup.py /work/datastore PFS 108361 108362 108363 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/verify.20241125a -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (108361, 108362, 108363) " --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 #defineVisitGroup.py /work/datastore PFS 108552 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run15/verify.20241125a -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit = 108552" --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20 ########### ## run14 ## ########### # PFS/calib/pipe2d-1547/run14/calibGen.20241203a ## bias ## pipetask run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/raw/sps,PFS/calib -o PFS/calib/pipe2d-1547/run14/biasGen.20241203a -p $DRP_STELLA_DIR/pipelines/bias.yaml -d "visit IN (103173..103197)" --fail-fast butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run14/calibGen.20241203a bias butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run14/biasGen.20241203a PFS/calib/pipe2d-1547/run14/calibGen.20241203a bias --begin-date 2000-01-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run14/biasGen.20241203a/* biasProc ## dark ## pipetask run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/darkGen.20241203a -p $DRP_STELLA_DIR/pipelines/dark.yaml -d "visit IN (103198..103222)" --fail-fast butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run14/calibGen.20241203a dark butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run14/darkGen.20241203a PFS/calib/pipe2d-1547/run14/calibGen.20241203a dark --begin-date 2000-01-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run14/darkGen.20241203a/* darkProc ## detectorMap based on run15 ## # brn pipetask --log-level .=INFO run --register-dataset-types -j 36 -skip-existing -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/brn -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (104144..104173)" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # m (run15 again) pipetask --log-level .=INFO run --register-dataset-types -j 36 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run15/calibGen.20241125a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/m-run15 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (106871..106902) AND arm = 'm'" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/m-run15 PFS/calib/pipe2d-1547/run14/calibGen.20241203a --instrument PFS --begin-date 2023-12-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 # m (run14) pipetask --log-level .=INFO run --register-dataset-types -j 36 -skip-existing -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/m -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (104176..104203) AND arm = 'm'" -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast ## bootstrap for b3, m1,m2,m3,m4 ## # b3: flatId=103142 arcId=103138 # m1: flatId=102564 arcId=102563 # m2: flatId=103157 arcId=103156 # m3: flatId=103144 arcId=103145 # m4: No data? # b3 pipetask --log-level .=INFO run --register-dataset-types -j 12 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/bootstrap/b3 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (103142, 103138) AND arm = 'b' AND spectrograph = 3" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c bootstrap:spatialOffset=0 -c bootstrap:spectralOffset=-10 -c bootstrap:allowSplit=False -C bootstrap:configs/sm3.config # m3 pipetask --log-level .=INFO run --register-dataset-types -j 12 -b $DATASTORE --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/bootstrap/m3 -p '$DRP_STELLA_DIR/pipelines/bootstrap.yaml' -d "visit IN (103144, 103145) AND arm = 'm' AND spectrograph = 3" --fail-fast -c isr:doCrosstalk=False -c bootstrap:profiles.profileRadius=2 -c bootstrap:profiles.profileSwath=2500 -c bootstrap:profiles.profileOversample=3 -c bootstrap:spatialOffset=5 -c bootstrap:spectralOffset=10 -c bootstrap:allowSplit=False -C bootstrap:configs/sm3.config butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run14/bootstrap/b3 PFS/bootstrap/run14 detectorMap_bootstrap butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run14/bootstrap/m3 PFS/bootstrap/run14 detectorMap_bootstrap # detectorMap based on bootstrap # b3 pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run14,PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/b3 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (104144..104159, 104170..104173, 104160..104169) AND spectrograph = 3 AND arm = 'b'" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # m3 pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/bootstrap/run14,PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/m3 -p '$DRP_STELLA_DIR/pipelines/detectorMap.yaml' -d "visit IN (104176..104203) AND spectrograph = 3 AND arm = 'm'" -c measureCentroids:connections.calibDetectorMap=detectorMap_bootstrap -c fitDetectorMap:fitDetectorMap.doSlitOffsets=True -c fitDetectorMap:fitDetectorMap.order=4 -c fitDetectorMap:fitDetectorMap.soften=0.03 --fail-fast # not improved much... # certify butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run14/calibGen.20241203a detectorMap_calib certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/brn PFS/calib/pipe2d-1547/run14/calibGen.20241203a --instrument PFS --begin-date 2000-01-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/m PFS/calib/pipe2d-1547/run14/calibGen.20241203a --instrument PFS --begin-date 2000-01-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run14/calibGen.20241203a detectorMap_calib --id instrument=PFS spectrograph=3 arm=b certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/b3 PFS/calib/pipe2d-1547/run14/calibGen.20241203a --instrument PFS --begin-date 2000-01-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 #butlerDecertify.py /work/datastore PFS/calib/pipe2d-1547/run14/calibGen.20241203a detectorMap_calib --id instrument=PFS spectrograph=3 arm=m #certifyDetectorMaps.py /work/datastore PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/m3 PFS/calib/pipe2d-1547/run14/calibGen.20241203a --instrument PFS --begin-date 2000-01-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run14/detectorMapsGen.20241203a/* postISRCCD ## fiberProfiles (failed?) ## # brn #defineFiberProfilesInputs.py /work/datastore PFS run14_brn --bright 104399..104407 --dark 104391..104397 --bright 104351..104359 --dark 104343..104349 --bright 104303..104311 --dark 104295..104301 --bright 104253..104261 --dark 104245..104251 --update defineFiberProfilesInputs.py /work/datastore PFS run14_brn_new --bright 104401..104407 --dark 104391..104397 --bright 104353..104359 --dark 104343..104349 --bright 104305..104311 --dark 104295..104301 --bright 104255..104261 --dark 104245..104251 --update pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run14/fiberProfilesGen.20241203a/brn -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run14_brn_new'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run14/fiberProfilesGen.20241203a/brn PFS/calib/pipe2d-1547/run14/calibGen.20241203a fiberProfiles --begin-date 2000-01-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 # m #defineFiberProfilesInputs.py /work/datastore PFS run14_m --bright 104447..104455 --dark 104439..104445 --bright 104527..104535 --dark 104519..104525 --bright 104578..104586 --dark 104570..104576 --bright 104626..104634 --dark 104618..104624 --update defineFiberProfilesInputs.py /work/datastore PFS run14_m_new --bright 104449..104455 --dark 104439..104445 --bright 104529..104535 --dark 104519..104525 --bright 104580..104586 --dark 104570..104576 --bright 104628..104634 --dark 104618..104624 --update pipetask --log-level .=INFO run --register-dataset-types -j 32 -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/fiberProfilesInputs,PFS/defaults -o PFS/calib/pipe2d-1547/run14/fiberProfilesGen.20241203a/m -p '$DRP_STELLA_DIR/pipelines/fitFiberProfiles.yaml' -d "profiles_run = 'run14_m_new' AND arm = 'm'" --fail-fast -c fitProfiles:profiles.profileRadius=10 -c fitProfiles:profiles.profileOversample=3 -c fitProfiles:profiles.profileSwath=500 butler certify-calibrations /work/datastore PFS/calib/pipe2d-1547/run14/fiberProfilesGen.20241203a/m PFS/calib/pipe2d-1547/run14/calibGen.20241203a fiberProfiles --begin-date 2000-01-01T00:00:00 --end-date 2024-02-28T23:59:59.9999 butlerCleanRun.py /work/datastore PFS/calib/pipe2d-1547/run14/fiberProfilesGen.20241203a/* postISRCCD # use fiberProfiles of run15 ? # ## fiberNorms (fails for m-band even if using run15 profiles) ## source ./generate_fiberNorms_pipe2d-1547_run14_20241203.sh ## verification ## defineVisitGroup.py /work/datastore PFS 108361 108362 108363 pipetask --log-level .=INFO run --register-dataset-types -b /work/datastore --instrument lsst.obs.pfs.PrimeFocusSpectrograph -i PFS/calib/pipe2d-1547/run14/calibGen.20241203a,PFS/defaults -o PFS/calib/pipe2d-1547/run14/verify.20241203a -p '$DRP_STELLA_DIR/pipelines/reduceExposure.yaml' -d "visit IN (104144..104173) " --fail-fast -j 32 -c reduceExposure:extractSpectra.minFracImage=0.2 -c mergeArms:fitSkyModel.blockSize=20