CIRR evaluation code

[hhoangcp]

Hi,
I am interested in reproducing the results from the CIRR dataset. Could you please share the evaluation code for CIRR?
I saw the CIRR evaluation results from your paper.
Thank you.

[drogozhang]

Hi, thanks for your interest. We haven't written the CIRR evaluation code for current implementation. It is just models transferred from inside parameters after testing on CIRCO, FashionIQ and DTIN.

I will let you know when I have the bandwidth to implement CIRR here.

[baohl00]

Hope my reproduced code helps you! @hhoangcp

def build_cirr_dataset(dataset_name: str, tokenizer: Any) -> Dataset:
    eval_dataset = Dataset(dataset_name)
    queries = json.load(open("./data/CIRR/cirr/captions/cap.rc2.test12.json"))
    coco_info = json.load(open("./data/CIRR/cirr/image_splits/split.rc2.test1.json"))
    index_img_ids = coco_info.keys() # [img_info['id'] for img_info in coco_info['images']]
    index_image_folder = "./data/CIRR/test1"
    def image_id2name(image_id):
        return image_id + '.png'
    null_tokens = tokenizer("")  # used for index example
    null_tokens = np.array(null_tokens)

    def process_index_example(index_img_id):
        img_path = os.path.join(index_image_folder, image_id2name(index_img_id))
        ima = process_img(img_path, 224)
        return IndexExample(iid=index_img_id, iimage=ima, itokens=null_tokens)

    def process_query_example(query):
        qid = query['pairid']
        qimage_path = os.path.join(index_image_folder, image_id2name(query['reference']))
        with open(qimage_path, 'rb') as f:
            img = PIL.Image.open(f).convert('RGB')
            image = preprocess(img).unsqueeze(0).to(device) 
        qtext = query['caption']
        ima = process_img(qimage_path, 224)
        qtokens = np.array(tokenizer(qtext))
        # circo test does not provide target id.
        return QueryExample(qid=qid, qtokens=qtokens, qimage=ima, target_iid=0, retrieved_iids=[], retrieved_scores=[])

    with ThreadPoolExecutor() as executor:
        print("Preparing index examples...")
        index_example_futures = {executor.submit(process_index_example, index_img_id): index_img_id for index_img_id in index_img_ids}

        with tqdm(total=len(index_img_ids), desc="Index examples") as progress:
            for future in as_completed(index_example_futures):
                index_example = future.result()
                eval_dataset.index_examples.append(index_example)
                progress.update(1)

        print("Prepared index examples.")

        print("Preparing query examples...")
        query_futures = {executor.submit(process_query_example, query): query for query in queries}

        with tqdm(total=len(queries), desc="Query examples") as progress:
            for future in as_completed(query_futures):
                q_example = future.result()
                eval_dataset.query_examples.append(q_example)
                progress.update(1)

    return eval_dataset

[ieellee]

@baohl00 I found that CIRR and CRICO recall metric all equal to 0, have you met this phenomenon? FIQ is normal.

[baohl00]

@baohl00 I found that CIRR and CRICO recall metric all equal to 0, have you met this phenomenon? FIQ is normal.

Everything's going well to me, although I couldn't reproduce exactly the same results.

Model	map@5	map@10	map@25	map@50
Base (original)	23.1	23.8	25.8	26.7
Base (reproduced)	25.5	26.5	28.5	29.4
Large (original)	29.6	30.8	33.4	34.4
Large (reproduced)	30.1	31.4	33.8	34.9