Pixel refinement

In this tutorial, we demostrate how to merge previous processed PS candidates in PS processing and DSs in DS processing together and refine these measurement points with Noise2Fringe Transformer.

import zarr
import numpy as np
from matplotlib import pyplot as plt
import colorcet
import moraine.cli as mc
import moraine as mr

Merge pixels

logger = mc.get_logger()
ds_ph = './ds_processing/hix/ds/ds_ph.zarr'
ds_e = './ds_processing/hix/ds/ds_e.zarr'
ds_n = './ds_processing/hix/ds/ds_n.zarr'
ds_hix = './ds_processing/hix/ds/ds_hix.zarr'

ps_can_rslc = './ps_processing/hix/ps_can_rslc.zarr'
ps_can_e = './ps_processing/hix/ps_can_e.zarr'
ps_can_n = './ps_processing/hix/ps_can_n.zarr'
ps_can_hix = './ps_processing/hix/ps_can_hix.zarr'

pc_can_hix = './pixel_refinement/hix/pc_can_hix.zarr'
pc_can_ph = './pixel_refinement/hix/pc_can_ph.zarr'
pc_can_e = './pixel_refinement/hix/pc_can_e.zarr'
pc_can_n = './pixel_refinement/hix/pc_can_n.zarr'
mc.pc_union(
    ps_can_hix, ds_hix, pc_can_hix,
    pc1 = [ps_can_rslc, ps_can_e, ps_can_n,],
    pc2 = [ds_ph, ds_e, ds_n],
    pc = [pc_can_ph, pc_can_e, pc_can_n],
)
2025-09-20 23:55:52 - log_args - INFO - running function: pc_union
2025-09-20 23:55:52 - log_args - INFO - fetching args:
2025-09-20 23:55:52 - log_args - INFO - idx1 = './ps_processing/hix/ps_can_hix.zarr'
2025-09-20 23:55:52 - log_args - INFO - idx2 = './ds_processing/hix/ds/ds_hix.zarr'
2025-09-20 23:55:52 - log_args - INFO - idx = './pixel_refinement/hix/pc_can_hix.zarr'
2025-09-20 23:55:52 - log_args - INFO - pc1 = ['./ps_processing/hix/ps_can_rslc.zarr', './ps_processing/hix/ps_can_e.zarr', './ps_processing/hix/ps_can_n.zarr']
2025-09-20 23:55:52 - log_args - INFO - pc2 = ['./ds_processing/hix/ds/ds_ph.zarr', './ds_processing/hix/ds/ds_e.zarr', './ds_processing/hix/ds/ds_n.zarr']
2025-09-20 23:55:52 - log_args - INFO - pc = ['./pixel_refinement/hix/pc_can_ph.zarr', './pixel_refinement/hix/pc_can_e.zarr', './pixel_refinement/hix/pc_can_n.zarr']
2025-09-20 23:55:52 - log_args - INFO - shape = None
2025-09-20 23:55:52 - log_args - INFO - chunks = None
2025-09-20 23:55:52 - log_args - INFO - processes = False
2025-09-20 23:55:52 - log_args - INFO - n_workers = 1
2025-09-20 23:55:52 - log_args - INFO - threads_per_worker = 1
2025-09-20 23:55:52 - log_args - INFO - dask_cluster_arg = {}
2025-09-20 23:55:52 - log_args - INFO - fetching args done.
2025-09-20 23:55:52 - zarr_info - INFO - ./ps_processing/hix/ps_can_hix.zarr zarray shape, chunks, dtype: (634220,), (200000,), int64
2025-09-20 23:55:52 - zarr_info - INFO - ./ds_processing/hix/ds/ds_hix.zarr zarray shape, chunks, dtype: (293814,), (200000,), int64
2025-09-20 23:55:52 - pc_union - INFO - loading idx1 and idx2 into memory.
2025-09-20 23:55:52 - pc_union - INFO - calculate the union
2025-09-20 23:55:52 - pc_union - INFO - number of points in the union: 758504
2025-09-20 23:55:52 - pc_union - INFO - write union idx
2025-09-20 23:55:52 - pc_union - INFO - write done
2025-09-20 23:55:52 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_hix.zarr zarray shape, chunks, dtype: (758504,), (200000,), int64
2025-09-20 23:55:52 - pc_union - INFO - starting dask local cluster.
2025-09-20 23:55:52 - pc_union - INFO - dask local cluster started.
2025-09-20 23:55:52 - dask_cluster_info - INFO - dask cluster: LocalCluster(dashboard_link='http://10.211.48.10:8787/status', workers=1, threads=1, memory=1.95 TiB)
2025-09-20 23:55:52 - zarr_info - INFO - ./ps_processing/hix/ps_can_rslc.zarr zarray shape, chunks, dtype: (634220, 17), (200000, 1), complex64
2025-09-20 23:55:52 - zarr_info - INFO - ./ds_processing/hix/ds/ds_ph.zarr zarray shape, chunks, dtype: (293814, 17), (200000, 1), complex64
2025-09-20 23:55:52 - darr_info - INFO - pc1 dask array shape, chunksize, dtype: (634220, 17), (634220, 1), complex64
2025-09-20 23:55:52 - darr_info - INFO - pc2 dask array shape, chunksize, dtype: (293814, 17), (293814, 1), complex64
2025-09-20 23:55:52 - pc_union - INFO - set up union pc data dask array.
2025-09-20 23:55:52 - darr_info - INFO - pc dask array shape, chunksize, dtype: (758504, 17), (758504, 1), complex64
2025-09-20 23:55:52 - pc_union - INFO - write pc to ./pixel_refinement/hix/pc_can_ph.zarr
2025-09-20 23:55:52 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_ph.zarr zarray shape, chunks, dtype: (758504, 17), (200000, 1), complex64
2025-09-20 23:55:52 - zarr_info - INFO - ./ps_processing/hix/ps_can_e.zarr zarray shape, chunks, dtype: (634220,), (200000,), float64
2025-09-20 23:55:52 - zarr_info - INFO - ./ds_processing/hix/ds/ds_e.zarr zarray shape, chunks, dtype: (293814,), (200000,), float64
2025-09-20 23:55:52 - darr_info - INFO - pc1 dask array shape, chunksize, dtype: (634220,), (634220,), float64
2025-09-20 23:55:52 - darr_info - INFO - pc2 dask array shape, chunksize, dtype: (293814,), (293814,), float64
2025-09-20 23:55:52 - pc_union - INFO - set up union pc data dask array.
2025-09-20 23:55:52 - darr_info - INFO - pc dask array shape, chunksize, dtype: (758504,), (758504,), float64
2025-09-20 23:55:52 - pc_union - INFO - write pc to ./pixel_refinement/hix/pc_can_e.zarr
2025-09-20 23:55:52 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_e.zarr zarray shape, chunks, dtype: (758504,), (200000,), float64
2025-09-20 23:55:52 - zarr_info - INFO - ./ps_processing/hix/ps_can_n.zarr zarray shape, chunks, dtype: (634220,), (200000,), float64
2025-09-20 23:55:52 - zarr_info - INFO - ./ds_processing/hix/ds/ds_n.zarr zarray shape, chunks, dtype: (293814,), (200000,), float64
2025-09-20 23:55:52 - darr_info - INFO - pc1 dask array shape, chunksize, dtype: (634220,), (634220,), float64
2025-09-20 23:55:52 - darr_info - INFO - pc2 dask array shape, chunksize, dtype: (293814,), (293814,), float64
2025-09-20 23:55:52 - pc_union - INFO - set up union pc data dask array.
2025-09-20 23:55:52 - darr_info - INFO - pc dask array shape, chunksize, dtype: (758504,), (758504,), float64
2025-09-20 23:55:52 - pc_union - INFO - write pc to ./pixel_refinement/hix/pc_can_n.zarr
2025-09-20 23:55:52 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_n.zarr zarray shape, chunks, dtype: (758504,), (200000,), float64
2025-09-20 23:55:52 - pc_union - INFO - computing graph setted. doing all the computing.
2025-09-20 23:55:53 - pc_union - INFO - computing finished. |  1.1s
2025-09-20 23:55:53 - pc_union - INFO - dask cluster closed.
pc_can_lon = './pixel_refinement/hix/pc_can_lon.zarr'
pc_can_lat = './pixel_refinement/hix/pc_can_lat.zarr'
mc.ras2pc(
    pc_can_hix, 
    ras=['./load_data/lon.zarr/','./load_data/lat.zarr/'],
    pc =[pc_can_lon, pc_can_lat]
)
2025-09-20 23:58:02 - log_args - INFO - running function: ras2pc
2025-09-20 23:58:02 - log_args - INFO - fetching args:
2025-09-20 23:58:02 - log_args - INFO - idx = './pixel_refinement/hix/pc_can_hix.zarr'
2025-09-20 23:58:02 - log_args - INFO - ras = ['./load_data/lon.zarr/', './load_data/lat.zarr/']
2025-09-20 23:58:02 - log_args - INFO - pc = ['./pixel_refinement/hix/pc_can_lon.zarr', './pixel_refinement/hix/pc_can_lat.zarr']
2025-09-20 23:58:02 - log_args - INFO - chunks = None
2025-09-20 23:58:02 - log_args - INFO - processes = False
2025-09-20 23:58:02 - log_args - INFO - n_workers = 1
2025-09-20 23:58:02 - log_args - INFO - threads_per_worker = 1
2025-09-20 23:58:02 - log_args - INFO - dask_cluster_arg = {}
2025-09-20 23:58:02 - log_args - INFO - fetching args done.
2025-09-20 23:58:02 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_hix.zarr zarray shape, chunks, dtype: (758504,), (200000,), int64
2025-09-20 23:58:02 - ras2pc - INFO - loading hix into memory and convert to gix
2025-09-20 23:58:04 - ras2pc - INFO - starting dask local cluster.
2025-09-20 23:58:04 - ras2pc - INFO - dask local cluster started.
2025-09-20 23:58:04 - dask_cluster_info - INFO - dask cluster: LocalCluster(dashboard_link='http://10.211.48.10:8787/status', workers=1, threads=1, memory=1.95 TiB)
2025-09-20 23:58:04 - ras2pc - INFO - start to slice on ./load_data/lon.zarr/
2025-09-20 23:58:04 - zarr_info - INFO - ./load_data/lon.zarr/ zarray shape, chunks, dtype: (2500, 1834), (1000, 1000), float64
2025-09-20 23:58:04 - darr_info - INFO - ras dask array shape, chunksize, dtype: (2500, 1834), (2500, 1834), float64
2025-09-20 23:58:04 - darr_info - INFO - pc dask array shape, chunksize, dtype: (758504,), (758504,), float64
2025-09-20 23:58:04 - ras2pc - INFO - saving to ./pixel_refinement/hix/pc_can_lon.zarr.
2025-09-20 23:58:04 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_lon.zarr zarray shape, chunks, dtype: (758504,), (200000,), float64
2025-09-20 23:58:04 - ras2pc - INFO - start to slice on ./load_data/lat.zarr/
2025-09-20 23:58:04 - zarr_info - INFO - ./load_data/lat.zarr/ zarray shape, chunks, dtype: (2500, 1834), (1000, 1000), float64
2025-09-20 23:58:04 - darr_info - INFO - ras dask array shape, chunksize, dtype: (2500, 1834), (2500, 1834), float64
2025-09-20 23:58:04 - darr_info - INFO - pc dask array shape, chunksize, dtype: (758504,), (758504,), float64
2025-09-20 23:58:04 - ras2pc - INFO - saving to ./pixel_refinement/hix/pc_can_lat.zarr.
2025-09-20 23:58:04 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_lat.zarr zarray shape, chunks, dtype: (758504,), (200000,), float64
2025-09-20 23:58:04 - ras2pc - INFO - computing graph setted. doing all the computing.
2025-09-20 23:58:05 - ras2pc - INFO - computing finished.ed |  0.2s
2025-09-20 23:58:05 - ras2pc - INFO - dask cluster closed.

Refine pixels with Noise2fringe Transformer

n2ft_intf = './pixel_refinement/hix/pc_can_n2f_intf.zarr'
pc_can_ph_zarr = zarr.open(pc_can_ph,mode='r')
tnet = tnet = mr.TempNet.from_bandwidth(pc_can_ph_zarr.shape[-1],1)
mc.n2ft(pc_can_lon, pc_can_lat, pc_can_ph, n2ft_intf, tnet.image_pairs, chunks= 20000, cuda=True)
2025-09-21 16:12:51 - log_args - INFO - running function: n2ft
2025-09-21 16:12:51 - log_args - INFO - fetching args:
2025-09-21 16:12:51 - log_args - INFO - x = './pixel_refinement/hix/pc_can_lon.zarr'
2025-09-21 16:12:51 - log_args - INFO - y = './pixel_refinement/hix/pc_can_lat.zarr'
2025-09-21 16:12:51 - log_args - INFO - rslc = './pixel_refinement/hix/pc_can_ph.zarr'
2025-09-21 16:12:51 - log_args - INFO - intf = './pixel_refinement/pc_can_n2f_intf.zarr'
2025-09-21 16:12:51 - log_args - INFO - image_pairs = array([[ 0,  1],
       [ 1,  2],
       [ 2,  3],
       [ 3,  4],
       [ 4,  5],
       [ 5,  6],
       [ 6,  7],
       [ 7,  8],
       [ 8,  9],
       [ 9, 10],
       [10, 11],
       [11, 12],
       [12, 13],
       [13, 14],
       [14, 15],
       [15, 16]], dtype=int32)
2025-09-21 16:12:51 - log_args - INFO - chunks = 20000
2025-09-21 16:12:51 - log_args - INFO - out_chunks = None
2025-09-21 16:12:51 - log_args - INFO - k = 128
2025-09-21 16:12:51 - log_args - INFO - model = None
2025-09-21 16:12:51 - log_args - INFO - cuda = True
2025-09-21 16:12:51 - log_args - INFO - processes = None
2025-09-21 16:12:51 - log_args - INFO - n_workers = None
2025-09-21 16:12:51 - log_args - INFO - threads_per_worker = None
2025-09-21 16:12:51 - log_args - INFO - rmm_pool_size = 0.5
2025-09-21 16:12:51 - log_args - INFO - dask_cluster_arg = {}
2025-09-21 16:12:51 - log_args - INFO - fetching args done.
2025-09-21 16:12:51 - n2ft - INFO - load coordinates
2025-09-21 16:12:51 - n2ft - INFO - Done
2025-09-21 16:12:51 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_ph.zarr zarray shape, chunks, dtype: (758504, 17), (200000, 1), complex64
2025-09-21 16:12:51 - n2ft - INFO - processing point chunk size: 20000
2025-09-21 16:12:51 - n2ft - INFO - distributing every processing chunk with halo data
2025-09-21 16:12:55 - n2ft - INFO - processing chunk size with halo data: [23208, 24455, 22837, 23069, 34383, 27737, 29607, 28344, 26739, 25523, 24083, 24445, 25315, 25322, 24721, 26577, 24715, 25789, 26051, 26238, 25108, 27963, 28192, 25935, 27746, 25604, 26038, 25298, 25983, 25076, 25275, 22929, 24983, 25265, 24195, 24766, 26729, 21755]
2025-09-21 16:12:55 - n2ft - INFO - starting dask cluster.
2025-09-21 16:13:04 - n2ft - INFO - dask cluster started.
2025-09-21 16:13:04 - dask_cluster_info - INFO - dask cluster: LocalCUDACluster(dashboard_link='http://127.0.0.1:8787/status', workers=8, threads=8, memory=1.95 TiB)
2025-09-21 16:13:04 - darr_info - INFO - rslc dask array shape, chunksize, dtype: (758504, 17), (758504, 1), complex64
2025-09-21 16:13:04 - n2ft - INFO - got filtered interferograms.
2025-09-21 16:13:04 - darr_info - INFO - intf dask array shape, chunksize, dtype: (758504, 16), (758504, 1), complex64
2025-09-21 16:13:04 - n2ft - INFO - saving filtered interferograms.
2025-09-21 16:13:04 - zarr_info - INFO - ./pixel_refinement/pc_can_n2f_intf.zarr zarray shape, chunks, dtype: (758504, 16), (200000, 1), complex64
2025-09-21 16:13:04 - n2ft - INFO - computing graph setted. doing all the computing.
[                                        ] | 0% Completed |  0.2s
/users/kangl/miniforge3/envs/work2/lib/python3.12/site-packages/distributed/client.py:3371: UserWarning: Sending large graph of size 24.85 MiB.
This may cause some slowdown.
Consider loading the data with Dask directly
 or using futures or delayed objects to embed the data into the graph without repetition.
See also https://docs.dask.org/en/stable/best-practices.html#load-data-with-dask for more information.
  warnings.warn(
2025-09-21 16:13:26 - n2ft - INFO - computing finished.eted | 21.7s
2025-09-21 16:13:30,258 - distributed.nanny - WARNING - Worker process still alive after 4.0 seconds, killing
2025-09-21 16:13:30 - n2ft - INFO - dask cluster closed.
temp_coh = './pixel_refinement/hix/pc_can_temp_coh.zarr'
mc.temp_coh(n2ft_intf, pc_can_ph, temp_coh, tnet.image_pairs, cuda=True)
2025-09-21 16:15:51 - log_args - INFO - running function: temp_coh
2025-09-21 16:15:51 - log_args - INFO - fetching args:
2025-09-21 16:15:51 - log_args - INFO - intf = './pixel_refinement/hix/pc_can_n2f_intf.zarr'
2025-09-21 16:15:51 - log_args - INFO - rslc = './pixel_refinement/hix/pc_can_ph.zarr'
2025-09-21 16:15:51 - log_args - INFO - t_coh = './pixel_refinement/hix/pc_can_temp_coh.zarr'
2025-09-21 16:15:51 - log_args - INFO - image_pairs = array([[ 0,  1],
       [ 1,  2],
       [ 2,  3],
       [ 3,  4],
       [ 4,  5],
       [ 5,  6],
       [ 6,  7],
       [ 7,  8],
       [ 8,  9],
       [ 9, 10],
       [10, 11],
       [11, 12],
       [12, 13],
       [13, 14],
       [14, 15],
       [15, 16]], dtype=int32)
2025-09-21 16:15:51 - log_args - INFO - chunks = None
2025-09-21 16:15:51 - log_args - INFO - cuda = True
2025-09-21 16:15:51 - log_args - INFO - processes = None
2025-09-21 16:15:51 - log_args - INFO - n_workers = None
2025-09-21 16:15:51 - log_args - INFO - threads_per_worker = None
2025-09-21 16:15:51 - log_args - INFO - rmm_pool_size = 0.9
2025-09-21 16:15:51 - log_args - INFO - dask_cluster_arg = {}
2025-09-21 16:15:51 - log_args - INFO - fetching args done.
2025-09-21 16:15:51 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_n2f_intf.zarr zarray shape, chunks, dtype: (758504, 16), (200000, 1), complex64
2025-09-21 16:15:51 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_ph.zarr zarray shape, chunks, dtype: (758504, 17), (200000, 1), complex64
2025-09-21 16:15:51 - temp_coh - INFO - starting dask local cluster.
2025-09-21 16:16:00 - temp_coh - INFO - dask local cluster started.
2025-09-21 16:16:00 - dask_cluster_info - INFO - dask cluster: LocalCUDACluster(dashboard_link='http://127.0.0.1:8787/status', workers=8, threads=8, memory=1.95 TiB)
2025-09-21 16:16:00 - darr_info - INFO - intf dask array shape, chunksize, dtype: (758504, 16), (200000, 16), complex64
2025-09-21 16:16:00 - darr_info - INFO - rslc dask array shape, chunksize, dtype: (758504, 17), (200000, 17), complex64
2025-09-21 16:16:00 - temp_coh - INFO - Estimate temporal coherence for DS.
2025-09-21 16:16:01 - temp_coh - INFO - got temporal coherence t_coh.
2025-09-21 16:16:01 - darr_info - INFO - t_coh dask array shape, chunksize, dtype: (758504,), (200000,), float32
2025-09-21 16:16:01 - temp_coh - INFO - saving t_coh.
2025-09-21 16:16:01 - temp_coh - INFO - computing graph setted. doing all the computing.
2025-09-21 16:16:02 - temp_coh - INFO - computing finished. |  0.9s
2025-09-21 16:16:03 - temp_coh - INFO - dask cluster closed.
import holoviews as hv
from bokeh.models import WheelZoomTool
hv.extension('bokeh')
import toml
with open('load_data/meta.toml','r') as f:
    dates = toml.load(f)['dates']
temp_coh_plot = mr.pc_plot(
    zarr.open(temp_coh, mode='r')[:],
    zarr.open(pc_can_n,mode='r')[:],
    zarr.open(pc_can_e,mode='r')[:],
)
ⓘ 
temp_coh_plot = temp_coh_plot.redim(
    x=hv.Dimension('lon', label='Longitude'), 
    y=hv.Dimension('lat',label='Latitude'), 
    z=hv.Dimension('temp_coh')
)
hv.output(widget_location='bottom')
hv.element.tiles.EsriImagery()*temp_coh_plot.opts(
    hv.opts.Image(
        cmap='fire',width=600, height=400, colorbar=True,
        default_tools=['pan',WheelZoomTool(zoom_on_axis=False),'save','reset','hover'],
        active_tools=['wheel_zoom']
    ),
    hv.opts.Points(
        color='temp_coh', cmap='fire',width=600, height=400, colorbar=True,
        default_tools=['pan',WheelZoomTool(zoom_on_axis=False),'save','reset','hover'],
        active_tools=['wheel_zoom']
    ),
)
pc_hix = './pixel_refinement/hix/pc/pc_hix.zarr'
mc.pc_logic_pc(pc_can_hix, temp_coh, pc_hix, 'pc_in>0.8')
2025-09-21 16:30:45 - log_args - INFO - running function: pc_logic_pc
2025-09-21 16:30:45 - log_args - INFO - fetching args:
2025-09-21 16:30:45 - log_args - INFO - idx_in = './pixel_refinement/hix/pc_can_hix.zarr'
2025-09-21 16:30:45 - log_args - INFO - pc_in = './pixel_refinement/hix/pc_can_temp_coh.zarr'
2025-09-21 16:30:45 - log_args - INFO - idx = './pixel_refinement/hix/pc/pc_hix.zarr'
2025-09-21 16:30:45 - log_args - INFO - operation = 'pc_in>0.8'
2025-09-21 16:30:45 - log_args - INFO - chunks = None
2025-09-21 16:30:45 - log_args - INFO - fetching args done.
2025-09-21 16:30:45 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_hix.zarr zarray shape, chunks, dtype: (758504,), (200000,), int64
2025-09-21 16:30:45 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_temp_coh.zarr zarray shape, chunks, dtype: (758504,), (200000,), float32
2025-09-21 16:30:45 - pc_logic_pc - INFO - loading idx_in into memory.
2025-09-21 16:30:45 - pc_logic_pc - INFO - loading pc_in into memory.
2025-09-21 16:30:45 - pc_logic_pc - INFO - select pc based on operation: pc_in>0.8
2025-09-21 16:30:45 - pc_logic_pc - INFO - number of selected pixels: 157318.
2025-09-21 16:30:45 - zarr_info - INFO - idx zarray shape, chunks, dtype: (157318,), (200000,), int64
2025-09-21 16:30:45 - pc_logic_pc - INFO - writing idx.
2025-09-21 16:30:45 - pc_logic_pc - INFO - write done.
pc_ph = './pixel_refinement/hix/pc/pc_ph.zarr'
pc_e = './pixel_refinement/hix/pc/pc_e.zarr'
pc_n = './pixel_refinement/hix/pc/pc_n.zarr'
mc.pc_select_data(
    pc_can_hix, pc_hix, 
    pc_in = [pc_can_ph, pc_can_e, pc_can_n],
    pc = [pc_ph, pc_e, pc_n],
)
2025-09-21 16:38:28 - log_args - INFO - running function: pc_select_data
2025-09-21 16:38:28 - log_args - INFO - fetching args:
2025-09-21 16:38:28 - log_args - INFO - idx_in = './pixel_refinement/hix/pc_can_hix.zarr'
2025-09-21 16:38:28 - log_args - INFO - idx = './pixel_refinement/hix/pc/pc_hix.zarr'
2025-09-21 16:38:28 - log_args - INFO - pc_in = ['./pixel_refinement/hix/pc_can_ph.zarr', './pixel_refinement/hix/pc_can_e.zarr', './pixel_refinement/hix/pc_can_n.zarr']
2025-09-21 16:38:28 - log_args - INFO - pc = ['./pixel_refinement/hix/pc/pc_ph.zarr', './pixel_refinement/hix/pc/pc_e.zarr', './pixel_refinement/hix/pc/pc_n.zarr']
2025-09-21 16:38:28 - log_args - INFO - shape = None
2025-09-21 16:38:28 - log_args - INFO - chunks = None
2025-09-21 16:38:28 - log_args - INFO - processes = False
2025-09-21 16:38:28 - log_args - INFO - n_workers = 1
2025-09-21 16:38:28 - log_args - INFO - threads_per_worker = 1
2025-09-21 16:38:28 - log_args - INFO - dask_cluster_arg = {}
2025-09-21 16:38:28 - log_args - INFO - fetching args done.
2025-09-21 16:38:28 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_hix.zarr zarray shape, chunks, dtype: (758504,), (200000,), int64
2025-09-21 16:38:28 - zarr_info - INFO - ./pixel_refinement/hix/pc/pc_hix.zarr zarray shape, chunks, dtype: (157318,), (200000,), int64
2025-09-21 16:38:28 - pc_select_data - INFO - loading idx_in and idx into memory.
2025-09-21 16:38:28 - pc_select_data - INFO - starting dask local cluster.
2025-09-21 16:38:28 - pc_select_data - INFO - dask local cluster started.
2025-09-21 16:38:28 - dask_cluster_info - INFO - dask cluster: LocalCluster(dashboard_link='http://10.211.48.10:8787/status', workers=1, threads=1, memory=1.95 TiB)
2025-09-21 16:38:28 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_ph.zarr zarray shape, chunks, dtype: (758504, 17), (200000, 1), complex64
2025-09-21 16:38:28 - darr_info - INFO - pc_in dask array shape, chunksize, dtype: (758504, 17), (758504, 1), complex64
2025-09-21 16:38:28 - pc_select_data - INFO - set up selected pc data dask array.
2025-09-21 16:38:28 - darr_info - INFO - pc dask array shape, chunksize, dtype: (157318, 17), (157318, 1), complex64
2025-09-21 16:38:28 - pc_select_data - INFO - write pc to ./pixel_refinement/hix/pc/pc_ph.zarr
2025-09-21 16:38:28 - zarr_info - INFO - ./pixel_refinement/hix/pc/pc_ph.zarr zarray shape, chunks, dtype: (157318, 17), (200000, 1), complex64
2025-09-21 16:38:28 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_e.zarr zarray shape, chunks, dtype: (758504,), (200000,), float64
2025-09-21 16:38:28 - darr_info - INFO - pc_in dask array shape, chunksize, dtype: (758504,), (758504,), float64
2025-09-21 16:38:28 - pc_select_data - INFO - set up selected pc data dask array.
2025-09-21 16:38:28 - darr_info - INFO - pc dask array shape, chunksize, dtype: (157318,), (157318,), float64
2025-09-21 16:38:28 - pc_select_data - INFO - write pc to ./pixel_refinement/hix/pc/pc_e.zarr
2025-09-21 16:38:28 - zarr_info - INFO - ./pixel_refinement/hix/pc/pc_e.zarr zarray shape, chunks, dtype: (157318,), (200000,), float64
2025-09-21 16:38:28 - zarr_info - INFO - ./pixel_refinement/hix/pc_can_n.zarr zarray shape, chunks, dtype: (758504,), (200000,), float64
2025-09-21 16:38:28 - darr_info - INFO - pc_in dask array shape, chunksize, dtype: (758504,), (758504,), float64
2025-09-21 16:38:28 - pc_select_data - INFO - set up selected pc data dask array.
2025-09-21 16:38:28 - darr_info - INFO - pc dask array shape, chunksize, dtype: (157318,), (157318,), float64
2025-09-21 16:38:28 - pc_select_data - INFO - write pc to ./pixel_refinement/hix/pc/pc_n.zarr
2025-09-21 16:38:28 - zarr_info - INFO - ./pixel_refinement/hix/pc/pc_n.zarr zarray shape, chunks, dtype: (157318,), (200000,), float64
2025-09-21 16:38:28 - pc_select_data - INFO - computing graph setted. doing all the computing.
2025-09-21 16:38:29 - pc_select_data - INFO - computing finished.5s
2025-09-21 16:38:29 - pc_select_data - INFO - dask cluster closed.

visualize the PS candidates

pc_ph_pyramid = './pixel_refinement/hix/pc/pc_ph_pyramid'
mc.pc_pyramid(
    pc_ph, pc_ph_pyramid,
    x = pc_e,
    y = pc_n,
    ras_resolution = 20,
)
2025-09-21 16:41:13 - log_args - INFO - running function: pc_pyramid
2025-09-21 16:41:13 - log_args - INFO - fetching args:
2025-09-21 16:41:13 - log_args - INFO - pc = './pixel_refinement/hix/pc/pc_ph.zarr'
2025-09-21 16:41:13 - log_args - INFO - out_dir = './pixel_refinement/hix/pc/pc_ph_pyramid'
2025-09-21 16:41:13 - log_args - INFO - x = './pixel_refinement/hix/pc/pc_e.zarr'
2025-09-21 16:41:13 - log_args - INFO - y = './pixel_refinement/hix/pc/pc_n.zarr'
2025-09-21 16:41:14 - log_args - INFO - yx = None
2025-09-21 16:41:14 - log_args - INFO - ras_resolution = 20
2025-09-21 16:41:14 - log_args - INFO - ras_chunks = (256, 256)
2025-09-21 16:41:14 - log_args - INFO - pc_chunks = 65536
2025-09-21 16:41:14 - log_args - INFO - processes = False
2025-09-21 16:41:14 - log_args - INFO - n_workers = 1
2025-09-21 16:41:14 - log_args - INFO - threads_per_worker = 2
2025-09-21 16:41:14 - log_args - INFO - dask_cluster_arg = {}
2025-09-21 16:41:14 - log_args - INFO - fetching args done.
2025-09-21 16:41:14 - pc_pyramid - INFO - clean out dir
2025-09-21 16:41:14 - zarr_info - INFO - ./pixel_refinement/hix/pc/pc_ph.zarr zarray shape, chunks, dtype: (157318, 17), (200000, 1), complex64
2025-09-21 16:41:14 - pc_pyramid - INFO - rendering point cloud data coordinates:
2025-09-21 16:41:14 - pc_pyramid - INFO - rasterizing point cloud data to grid with bounds: [np.float64(-16498472.188209932), np.float64(8649654.067951033), np.float64(-16470212.188209932), np.float64(8674674.067951033)].
2025-09-21 16:41:14 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/x.zarr zarray shape, chunks, dtype: (157318,), (65536,), float64
2025-09-21 16:41:14 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/y.zarr zarray shape, chunks, dtype: (157318,), (65536,), float64
2025-09-21 16:41:14 - pc_pyramid - INFO - pc data coordinates rendering ends.
2025-09-21 16:41:14 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_0.zarr zarray shape, chunks, dtype: (1252, 1414), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_1.zarr zarray shape, chunks, dtype: (626, 707), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_2.zarr zarray shape, chunks, dtype: (313, 354), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_3.zarr zarray shape, chunks, dtype: (157, 177), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_4.zarr zarray shape, chunks, dtype: (79, 89), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_5.zarr zarray shape, chunks, dtype: (40, 45), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_6.zarr zarray shape, chunks, dtype: (20, 23), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_7.zarr zarray shape, chunks, dtype: (10, 12), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_8.zarr zarray shape, chunks, dtype: (5, 6), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_9.zarr zarray shape, chunks, dtype: (3, 3), (256, 256), int64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/idx_10.zarr zarray shape, chunks, dtype: (2, 2), (256, 256), int64
2025-09-21 16:41:15 - pc_pyramid - INFO - rasterized idx rendering ends
2025-09-21 16:41:15 - pc_pyramid - INFO - dask local cluster started to render pc data.
2025-09-21 16:41:15 - dask_cluster_info - INFO - dask cluster: LocalCluster(dashboard_link='http://10.211.48.10:8787/status', workers=1, threads=2, memory=1.95 TiB)
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/pc.zarr zarray shape, chunks, dtype: (157318, 17), (65536, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/0.zarr zarray shape, chunks, dtype: (1252, 1414, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/1.zarr zarray shape, chunks, dtype: (626, 707, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/2.zarr zarray shape, chunks, dtype: (313, 354, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/3.zarr zarray shape, chunks, dtype: (157, 177, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/4.zarr zarray shape, chunks, dtype: (79, 89, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/5.zarr zarray shape, chunks, dtype: (40, 45, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/6.zarr zarray shape, chunks, dtype: (20, 23, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/7.zarr zarray shape, chunks, dtype: (10, 12, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/8.zarr zarray shape, chunks, dtype: (5, 6, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/9.zarr zarray shape, chunks, dtype: (3, 3, 17), (256, 256, 1), complex64
2025-09-21 16:41:15 - zarr_info - INFO - pixel_refinement/hix/pc/pc_ph_pyramid/10.zarr zarray shape, chunks, dtype: (2, 2, 17), (256, 256, 1), complex64
2025-09-21 16:41:16 - pc_pyramid - INFO - computing graph setted. doing all the computing.
2025-09-21 16:41:18 - pc_pyramid - INFO - computing finished.  1.9s
2025-09-21 16:41:18 - pc_pyramid - INFO - dask cluster closed.
intf_plot = mc.pc_plot(pc_ph_pyramid,post_proc_ras='intf_all', post_proc_pc='intf_all',level_increase=0)
import holoviews as hv
from bokeh.models import WheelZoomTool
hv.extension('bokeh')
import toml
with open('load_data/meta.toml','r') as f:
    dates = toml.load(f)['dates']
intf_plot = intf_plot.redim(i=hv.Dimension('i', label='Reference Image', range=(0,16), value_format=(lambda i: dates[i])),
                            j=hv.Dimension('j', label='Secondary Image', range=(0,16), value_format=(lambda i: dates[i])),
                            x=hv.Dimension('lon', label='Longitude'), y=hv.Dimension('lat',label='Latitude'), z=hv.Dimension('Phase',range=(-np.pi,np.pi)))
ⓘ 
hv.output(widget_location='bottom')
hv.element.tiles.EsriImagery()*intf_plot.opts(
    hv.opts.Image(
        cmap='colorwheel',width=600, height=400, colorbar=True,
        default_tools=['pan',WheelZoomTool(zoom_on_axis=False),'save','reset','hover'],
        active_tools=['wheel_zoom']
    ),
    hv.opts.Points(
        color='Phase', cmap='colorwheel',width=600, height=400, colorbar=True,
        default_tools=['pan',WheelZoomTool(zoom_on_axis=False),'save','reset','hover'],
        active_tools=['wheel_zoom']
    ),
)