U
    h a                     @  s  U d dl mZ d dlZd dlmZmZmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlZdddddZeaeaeadddddZ ej!dddddZ"ej#dddddZ$ej#ddddd Z%dddd!d"d#Z&dddd!d$d%Z'ddd&d!d'd(Z(G d)d* d*ej)Z*G d+d, d,ej+Z,G d-d. d.ej-Z.G d/d0 d0ej/Z0G d1d2 d2ej1Z2d3d4d5giZ3e. Z4da5d6d7gZ6da7d8e8d9< e0 a9d:d;d<d=Z:G d>d? d?ej;Z<e	j=>ej?d@ej?e@ iG dAdB dBe<ZAe	j=>ej?d@ej?e@ iG dCdD dDe<ZBe	j=>ej?d@ej?e@ iG dEdF dFe<ZCe	j=>ej?d@ej?e@ iG dGdH dHe<ZDe	j=>ej?d@ej?e@ iG dIdJ dJe<ZEe@d@kreF  dS )K    )annotationsN)AnyCallableMappingSequence)mock)logging)absltest)gspread_client)ipython_env)magics_engine)model_registry)post_process_utils)	sheets_id)sheets_utils)llm_function)llmfn_inputs_source)llmfn_outputs)modelr   xreturnc                 C  s   dst d| S )NFzShould not be called)AssertionError)r    r   S/tmp/pip-unpacked-wheel-doshhd5e/google/generativeai/notebook/magics_engine_test.py_fake_llm_function%   s    r   strint)resultr   c                 C  s   t | S Nlenr   r   r   r   
add_length1   s    r#   c                 C  s   t | S r   r    r"   r   r   r   add_length_decorated5   s    r$   c                 C  s   | |  S r   r   r"   r   r   r   repeat:   s    r%   c                 C  s   |   S r   )upperr"   r   r   r   to_upper?   s    r'   )lhsrhsr   c                 C  s   t | t | S r   r    r(   r)   r   r   r   get_sum_of_lengthsE   s    r+   c                 C  s   | d | S )N r   r*   r   r   r   concatI   s    r-   boolc                 C  s   | |kS r   r   r*   r   r   r   my_is_equal_fnM   s    r/   c                   @  s*   e Zd ZdZd
ddZddddd	ZdS )EchoModelRegistryz Fake model registry for testing.Nc                 C  s   |p
t  | _ d | _d S r   )r   	EchoModelget_model_name)selfZ	alt_modelr   r   r   __init__T   s    zEchoModelRegistry.__init__zmodel_registry.ModelNamezmodel.AbstractModel)
model_namer   c                 C  s   || _ | jS r   )r2   r   )r3   r5   r   r   r   	get_modelX   s    zEchoModelRegistry.get_model)N)__name__
__module____qualname____doc__r4   r6   r   r   r   r   r0   Q   s   
r0   c                   @  sF   e Zd ZdZdd ZddddZddd	d
dZddd	ddZdS )FakeIPythonEnvzFake IPythonEnv for testing.c                 C  s
   d | _ d S r   display_argsr3   r   r   r   r4   b   s    zFakeIPythonEnv.__init__Noner   c                 C  s
   d | _ d S r   r<   r>   r   r   r   cleare   s    zFakeIPythonEnv.clearr   r   c                 C  s   || _ td| d S )Nz"IPythonEnv.display called with:
%r)r=   r   infor3   r   r   r   r   displayh   s    zFakeIPythonEnv.displayc                 C  s   t d| d S )Nz'IPythonEnv.display_html called with:
%r)r   rB   rC   r   r   r   display_htmll   s    zFakeIPythonEnv.display_htmlN)r7   r8   r9   r:   r4   rA   rD   rE   r   r   r   r   r;   _   s
   r;   c                   @  s   e Zd ZddddZdS )FakeInputsSource6tuple[Sequence[Mapping[str, str]], Callable[[], None]]r@   c                 C  s   ddiddigdd fS )NwordZquack3Zquack4c                   S  s   d S r   r   r   r   r   r   <lambda>u       z=FakeInputsSource._to_normalized_inputs_impl.<locals>.<lambda>r   r>   r   r   r   _to_normalized_inputs_implr   s    z+FakeInputsSource._to_normalized_inputs_implN)r7   r8   r9   rK   r   r   r   r   rF   p   s   rF   c                   @  s$   e Zd Zdd ZdddddZdS )	FakeOutputsSinkc                 C  s
   d | _ d S r   outputsr>   r   r   r   r4   z   s    zFakeOutputsSink.__init__zllmfn_outputs.LLMFnOutputsBaser?   )rN   r   c                 C  s
   || _ d S r   rM   )r3   rN   r   r   r   write_outputs}   s    zFakeOutputsSink.write_outputsN)r7   r8   r9   r4   rO   r   r   r   r   rL   x   s   rL   c                   @  sF   e Zd Zdd ZddddZdddd	d
dZddddddZdS )MockGSpreadClientc                 C  s   d | _ d | _d | _d | _d S r   )get_all_records_nameget_all_records_worksheet_idwrite_records_namewrite_records_rowsr>   r   r   r   r4      s    zMockGSpreadClient.__init__zsheets_id.SheetsIdentifier)sidc                 C  s    |  d krtd|d S )NzSheets not found: {})namer
   ZSpreadsheetNotFoundErrorformat)r3   rU   r   r   r   validate   s
    zMockGSpreadClient.validater   rG   )rU   worksheet_idr   c                 C  s(   |  | _|| _ddiddigdd fS )NrH   Zquack5Zquack6c                   S  s   d S r   r   r   r   r   r   rI      rJ   z3MockGSpreadClient.get_all_records.<locals>.<lambda>)rV   rQ   rR   )r3   rU   rY   r   r   r   get_all_records   s    
z!MockGSpreadClient.get_all_recordszSequence[Sequence[Any]]r?   )rU   rowsr   c                 C  s   |  | _|| _d S r   )rV   rS   rT   )r3   rU   r[   r   r   r   write_records   s    
zMockGSpreadClient.write_recordsN)r7   r8   r9   r4   rX   rZ   r\   r   r   r   r   rP      s   	rP   rH   quack1quack2QUACK QUACK1NOT QUACK QUACK2z!llmfn_outputs.LLMFnOutputs | None_output_varr?   r@   c                   C  s0   d a d ad ad at atjtj	dddda
d S )Nfake_sheets)rV   *   )rU   rY   )_compiled_function_compiled_lhs_function_compiled_rhs_functionra   rL   _output_sink_varr   ZSheetsInputsr   ZSheetsIdentifier_SHEETS_INPUT_VARr   r   r   r   _reset_globals   s    	
ri   c                      sB   e Zd Z fddZddddddZd	dd
ddddZ  ZS )EndToEndTestsc                   s(   t    t | _t| j t  d S r   )supersetUprP   _mock_clientr
   Ztestonly_set_clientri   r>   	__class__r   r   rl      s    
zEndToEndTests.setUpzpandas.DataFramezMapping[str, Any]r?   )resultsexpected_resultsr   c                 C  sJ   |  |tj | ||jdd | t| t|jdd  d S )NlistZorient)assertIsInstancepandas	DataFrameassertEqualto_dictrr   keys)r3   rp   rq   r   r   r   $_assert_is_expected_pandas_dataframe   s    
z2EndToEndTests._assert_is_expected_pandas_dataframer   r;   )varrq   fake_envr   c                 C  s   |  |tj | ||  | t| t|   | t|d |	  |
  |  |jtj | ||jjdd d S )N_ipython_display_rr   rs   )rt   r   LLMFnOutputsrw   Zas_dictrr   ry   
assertTruehasattrrA   r}   r=   ru   rv   rx   )r3   r{   rq   r|   r   r   r   &_assert_output_var_is_expected_results   s    
 z4EndToEndTests._assert_output_var_is_expected_results)r7   r8   r9   rl   rz   r   __classcell__r   r   rn   r   rj      s   
rj   __main__c                   @  s$   e Zd Zdd Zdd Zdd ZdS )HelpEndToEndTestsc                 C  s6   t  }d}tj|d}||d}| t|d d S )Nz--helpregistryignoredz"A system for interacting with LLMsr0   r   MagicsEngineexecute_cellassertRegexr   r3   mock_registry
magic_lineenginerp   r   r   r   	test_help   s
    zHelpEndToEndTests.test_helpc                 C  s6   t  }d}tj|d}||d}| t|d d S )Nz
run --helpr   r   zusage: palm runr   r   r   r   r   test_run_help   s
    zHelpEndToEndTests.test_run_helpc              	   C  s>   t  }d}tj|d}| td ||d W 5 Q R X d S )Nzrun --this_is_an_invalid_flagr   z1unrecognized arguments: --this_is_an_invalid_flagr   )r0   r   r   assertRaisesRegex
SystemExitr   )r3   r   r   r   r   r   r   
test_error  s     zHelpEndToEndTests.test_errorN)r7   r8   r9   r   r   r   r   r   r   r   r      s   		r   c                   @  sl   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd ZdS )RunCmdEndToEndTestsc                 C  s   t  }d}tj|d}||d}dgdgdgdgdgd}| j||d ||d}dgdgdgdgdgd}| j||d ||d	}dgdgdgdgdgd}| j||d | tjj|j	 d
S )z)Smoke test for executing the run command.zrun --model_type=echor   zquack quackr   
Prompt Num	Input Num
Result NumPrompttext_resultrp   rq   zline 1
line 2
zline 1
line 2zline 1
line 2

N)
r0   r   r   r   rz   rw   r   Z	ModelNameZ
ECHO_MODELr2   )r3   r   r   r   rp   rq   r   r   r   test_run_cmd  sP        z RunCmdEndToEndTests.test_run_cmdc                 C  sZ   t tj}t|}tj|d}|dd}tjdddd}|j	j
jd }| || d S )	Nr   zSrun --model_type=echo --model=the_best_model --temperature=0.25 --candidate_count=3quackZthe_best_modelg      ?   )r   ZtemperatureZcandidate_countZ
model_args)r   Zcreate_autospecr   r1   r0   r   r   r   ZModelArgumentsZ
call_modelZ	call_argskwargsrw   )r3   Z
mock_modelregr   _Zexpected_model_argsZactual_model_argsr   r   r   test_model_args_passedG  s      z*RunCmdEndToEndTests.test_model_args_passedc                 C  s^   t  }tj|d}|dd}dddgdddgdddgdddgdddgd}| j||d d S )	Nr   z#run --model_type=echo --candidate=3r   r         r   r   r0   r   r   r   rz   r3   r   r   rp   rq   r   r   r   test_candidate_countZ  s      z(RunCmdEndToEndTests.test_candidate_countc                 C  sJ   t  }tj|d}|dd}dgdgdgdgdgd}| j||d d S )Nr   z,run --model_type=echo --candidate=3 --uniquer   r   r   r   r   r   r   r   r   test_uniquel  s      zRunCmdEndToEndTests.test_uniquec              
   C  s   d}t jt d}||d}| |tj ddddddgdddddd	gddddddgd
dddddgd
dddddgd}| j||d | d| j	j
 | d| j	j d S )NzNrun --model_type=echo --inputs _INPUT_VAR_ONE _INPUT_VAR_TWO _SHEETS_INPUT_VARr   quack {word}r   r   r   r         zquack quack1zquack quack2zquack quack3zquack quack4quack quack5quack quack6r   r   rb   rc   r   r   r0   r   rt   ru   rv   rz   rw   rm   rQ   rR   r3   r   r   rp   rq   r   r   r   test_inputs_passed~  s:    	 z&RunCmdEndToEndTests.test_inputs_passedc                 C  s   d}t jt d}||d}| |tj ddgddgddgddgddgd}| j||d	 | d
| j	j
 | d| j	j dS )z Test using --sheets_input_names.9run --model_type=echo --sheets_input_names my_fake_sheetsr   r   r   r   r   r   r   r   Zmy_fake_sheetsNr   r   r   r   r   test_sheets_input_names_passed  s(     z2RunCmdEndToEndTests.test_sheets_input_names_passedc              	   C  s~   t jt d}| td |dd W 5 Q R X | td |dd W 5 Q R X | td |dd W 5 Q R X d S )	Nr   zrargument --inputs/-i: Error with value "_INPUT_VAR_ONE", got ValueError: Placeholder "not_word" not found in inputz-run --model_type=echo --inputs _INPUT_VAR_ONEquack {not_word}zrargument --inputs/-i: Error with value "_INPUT_VAR_TWO", got ValueError: Placeholder "not_word" not found in inputz-run --model_type=echo --inputs _INPUT_VAR_TWOzuargument --inputs/-i: Error with value "_SHEETS_INPUT_VAR", got ValueError: Placeholder "not_word" not found in inputz0run --model_type=echo --inputs _SHEETS_INPUT_VARr   r   r0   r   r   r   r3   r   r   r   r   )test_validate_inputs_against_placeholders  s2       z=RunCmdEndToEndTests.test_validate_inputs_against_placeholdersc              	   C  s6   t jt d}| td |dd W 5 Q R X d S )Nr   zargument --sheets_input_names/-si: Error with value "my_fake_sheets", got ValueError: Placeholder "not_word" not found in inputr   r   r   r   r   r   r   0test_validate_sheets_inputs_against_placeholders  s    zDRunCmdEndToEndTests.test_validate_sheets_inputs_against_placeholdersc                 C  s`   d}t jt d}||d}| |tj dgdgdgdgdgdgdgd}| j||d	 d S )
NzBrun --model_type=echo | add_length | repeat | add_length_decoratedr   r   r   r   
   
quackquack)r   r   r   r   r#   r$   r   r   )r   r   r0   r   rt   ru   rv   rz   r   r   r   r   test_post_process  s"    	 z%RunCmdEndToEndTests.test_post_processc                 C  st   d}t  }tjt |d}||d}| ttj dgdgdgdgdgdgd}| j	||d | j
t||d	 d S )
NzArun --model_type=echo --outputs _output_var | add_length | repeatr   envr   r   r   r   r   r   r   r   r#   r   r   r{   rq   r|   )r;   r   r   r0   r   rt   ra   r   r~   rz   r   )r3   r   r|   r   rp   rq   r   r   r   test_outputs  s2        z RunCmdEndToEndTests.test_outputsc                 C  s   d}t jt d}||d}| tj | tjtj	 dgdgdgdgdgdgd}| j
||d | |tj jd	d
 d S )NzFrun --model_type=echo --outputs _output_sink_var | add_length | repeatr   r   r   r   r   r   r   rr   rs   )r   r   r0   r   ZassertIsNotNonerg   rN   rt   r   r~   rz   rw   Zas_pandas_dataframerx   r   r   r   r   test_outputs_sink!  s*     z%RunCmdEndToEndTests.test_outputs_sinkc                 C  sb   d}t jt d}||d}| d| jj ddddd	d
gddddddgg}| || jj d S )NzXrun --model_type=echo --sheets_output_names my_fake_output_sheets  | add_length | repeatr   r   Zmy_fake_output_sheetsr   r   r   r   r#   r   r   r   r   )r   r   r0   r   rw   rm   rS   rT   )r3   r   r   r   Zexpected_rowsr   r   r   test_sheets_outputs_names<  s$     z-RunCmdEndToEndTests.test_sheets_outputs_namesN)r7   r8   r9   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r     s   0$#r   c                   @  s   e Zd Zdd ZdS )CompileCmdEndToEndTestsc                 C  sl   t  }tjt |d}|dd}| ttj tddgi}dgdgdgdgdgd}| j	|||d	 d S )
Nr   z,compile _compiled_function --model_type=echor   rH   z
LOUD QUACKr   zquack LOUD QUACKr   r   )
r;   r   r   r0   r   rt   rd   r   ZLLMFunctionr   )r3   r|   r   r   rN   Zexpected_outputsr   r   r   test_compile_cmd[  s,        z(CompileCmdEndToEndTests.test_compile_cmdN)r7   r8   r9   r   r   r   r   r   r   X  s   r   c                   @  s   e Zd Zdd Zdd ZdS )CompareCmdEndToEndTestsc                 C  s   t  }tjt |d}|dd}|dd}|dd}ddgdd	gddgd
did
digddgddgddgd}| j||d | jt||d d S )Nr   0compile _compiled_lhs_function --model_type=echoleft quack {word}0compile _compiled_rhs_function --model_type=echoright quack {word}zccompare _compiled_lhs_function _compiled_rhs_function --inputs _INPUT_VAR_ONE --outputs _output_varr   r   r   rH   r]   r^   left quack quack1left quack quack2right quack quack1right quack quack2F)r   r   r   Prompt vars"_compiled_lhs_function_text_result"_compiled_rhs_function_text_resultZis_equalr   r   r;   r   r   r0   r   rz   r   ra   r3   r|   r   r   rp   rq   r   r   r   (test_compare_cmd_with_default_compare_fny  sL       
   z@CompareCmdEndToEndTests.test_compare_cmd_with_default_compare_fnc              	   C  s   t  }tjt |d}|dd}|dd}|dd}ddgdd	gddgd
did
digddgddgddgddgd}| j||d | jt||d d S )Nr   r   r   r   r   zcompare _compiled_lhs_function _compiled_rhs_function --inputs _INPUT_VAR_ONE --outputs _output_var --compare_fn concat get_sum_of_lengthsr   r   r   rH   r]   r^   r   r   r   r   z$left quack quack1 right quack quack1z$left quack quack2 right quack quack2#   )r   r   r   r   r   r   r-   r+   r   r   r   r   r   r   r   'test_compare_cmd_with_custom_compare_fn  sR          z?CompareCmdEndToEndTests.test_compare_cmd_with_custom_compare_fnN)r7   r8   r9   r   r   r   r   r   r   r   v  s   /r   c                   @  s   e Zd Zdd ZdS )EvalCmdEndToEndTestsc              	   C  s   t  }tjt |d}|dd}ddgddgddgddiddigd	d
gd	dgddgddgd}| j||d | jt||d d S )Nr   zeval --model_type=echo --ground_truth _GROUND_TRUTH_VAR --inputs _INPUT_VAR_ONE --outputs _output_var --compare_fn get_sum_of_lengths my_is_equal_fn | to_upperr   r   r   rH   r]   r^   r_   zQUACK QUACK2r`         TF)r   r   r   r   Zactual_text_resultZground_truth_text_resultr+   r/   r   r   r   )r3   r|   r   rp   rq   r   r   r   test_eval_cmd  s6     

   z"EvalCmdEndToEndTests.test_eval_cmdN)r7   r8   r9   r   r   r   r   r   r     s   r   )G
__future__r   systypingr   r   r   r   Zunittestr   Zabslr   Zabsl.testingr	   Zgoogle.generativeai.notebookr
   r   r   r   r   r   r   Z google.generativeai.notebook.libr   r   r   r   ru   r   rd   re   rf   r#   Zpost_process_add_fnr$   Zpost_process_replace_fnr%   r'   r+   r-   r/   ZModelRegistryr0   Z
IPythonEnvr;   ZLLMFnInputsSourcerF   ZLLMFnOutputsSinkrL   ZGSpreadClientrP   Z_INPUT_VAR_ONEZ_INPUT_VAR_TWOrh   Z_GROUND_TRUTH_VARra   __annotations__rg   ri   ZTestCaserj   patchdictmodulesr7   r   r   r   r   r   mainr   r   r   r   <module>   sv    	#-   Eg+
