
    5P@i i                       U d dl mZ d dlZd dlmZmZmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlZd9dZeaeaead:dZ ejB                  d:d       Z"ejF                  d;d       Z$ejF                  d;d       Z%d<dZ&d=dZ'd>dZ( G d dejR                        Z* G d dejV                        Z, G d dejZ                        Z. G d  d!ej^                        Z0 G d" d#ejb                        Z2d$d%d&giZ3 e.       Z4da5d'd(gZ6da7d)e8d*<    e0       a9d?d+Z: G d, d-ejv                        Z<e	jz                  j}                  ej~                  d.ej~                  e@   i       G d/ d0e<             ZAe	jz                  j}                  ej~                  d.ej~                  e@   i       G d1 d2e<             ZBe	jz                  j}                  ej~                  d.ej~                  e@   i       G d3 d4e<             ZCe	jz                  j}                  ej~                  d.ej~                  e@   i       G d5 d6e<             ZDe	jz                  j}                  ej~                  d.ej~                  e@   i       G d7 d8e<             ZEe@d.k(  r ej                          yy)@    )annotationsN)AnyCallableMappingSequence)mock)logging)absltest)gspread_client)ipython_env)magics_engine)model_registry)post_process_utils)	sheets_id)sheets_utils)llm_function)llmfn_inputs_source)llmfn_outputs)modelc                    J d       )NzShould not be called )xs    y/home/www/lebenam.kofcorporation.com/venv/lib/python3.12/site-packages/google/generativeai/notebook/magics_engine_test.py_fake_llm_functionr   %   s    (((5    c                    t        |       S Nlenresults    r   
add_lengthr"   1   s    v;r   c                    t        |       S r   r   r    s    r   add_length_decoratedr$   5   s    v;r   c                    | | z   S r   r   r    s    r   repeatr&   :   s    F?r   c                "    | j                         S r   )upperr    s    r   to_upperr)   ?   s    <<>r   c                0    t        |       t        |      z   S r   r   lhsrhss     r   get_sum_of_lengthsr.   E   s    s8c#hr   c                    | dz   |z   S )N r   r+   s     r   concatr1   I   s    9s?r   c                    | |k(  S r   r   r+   s     r   my_is_equal_fnr3   M   s    #:r   c                       e Zd ZdZddZddZy)EchoModelRegistryz Fake model registry for testing.Nc                L    |xs t        j                         | _         d | _        y r   )r   	EchoModelget_model_name)self	alt_models     r   __init__zEchoModelRegistry.__init__T   s    3%//"3
?Cr   c                (    || _         | j                  S r   )r8   r   )r9   
model_names     r   	get_modelzEchoModelRegistry.get_modelX   s    (zzr   r   )r=   zmodel_registry.ModelNamereturnzmodel.AbstractModel)__name__
__module____qualname____doc__r;   r>   r   r   r   r5   r5   Q   s    *Dr   r5   c                  .    e Zd ZdZd ZddZddZddZy)	FakeIPythonEnvzFake IPythonEnv for testing.c                    d | _         y r   display_argsr9   s    r   r;   zFakeIPythonEnv.__init__`   s
    !%r   c                    d | _         y r   rG   rI   s    r   clearzFakeIPythonEnv.clearc   s
     r   c                >    || _         t        j                  d|       y )Nz"IPythonEnv.display called with:
%r)rH   r	   infor9   r   s     r   displayzFakeIPythonEnv.displayf   s    :A>r   c                0    t        j                  d|       y )Nz'IPythonEnv.display_html called with:
%r)r	   rM   rN   s     r   display_htmlzFakeIPythonEnv.display_htmlj   s    ?Cr   Nr?   None)r   r   r?   rS   )r@   rA   rB   rC   r;   rK   rO   rQ   r   r   r   rE   rE   ]   s    &&!?Dr   rE   c                      e Zd Z	 	 ddZy)FakeInputsSourcec                    ddiddigd fS )Nwordquack3quack4c                      y r   r   r   r   r   <lambda>z=FakeInputsSource._to_normalized_inputs_impl.<locals>.<lambda>r       r   r   rI   s    r   _to_normalized_inputs_implz+FakeInputsSource._to_normalized_inputs_implo   s     "VX$67EEr   N)r?   6tuple[Sequence[Mapping[str, str]], Callable[[], None]])r@   rA   rB   r]   r   r   r   rU   rU   n   s    F	?Fr   rU   c                      e Zd Zd ZddZy)FakeOutputsSinkc                    d | _         y r   outputsrI   s    r   r;   zFakeOutputsSink.__init__v   s	    >Br   c                    || _         y r   rb   )r9   rc   s     r   write_outputszFakeOutputsSink.write_outputsy   s	    r   N)rc   zllmfn_outputs.LLMFnOutputsBaser?   rS   )r@   rA   rB   r;   re   r   r   r   r`   r`   u   s    Cr   r`   c                  B    e Zd Zd ZddZ	 	 	 	 	 	 ddZ	 	 	 	 	 	 ddZy)	MockGSpreadClientc                <    d | _         d | _        d | _        d | _        y r   )get_all_records_nameget_all_records_worksheet_idwrite_records_namewrite_records_rowsrI   s    r   r;   zMockGSpreadClient.__init__~   s"    04!8<).2BFr   c                l    |j                         $t        j                  dj                  |            y )NzSheets not found: {})namer   SpreadsheetNotFoundErrorformat)r9   sids     r   validatezMockGSpreadClient.validate   s/    88: 99:P:W:WX[:\]]r   c                P    |j                         | _        || _        ddiddigd fS )NrW   quack5quack6c                      y r   r   r   r   r   r[   z3MockGSpreadClient.get_all_records.<locals>.<lambda>   r\   r   )rn   ri   rj   )r9   rq   worksheet_ids      r   get_all_recordsz!MockGSpreadClient.get_all_records   s4    
 %(HHJ!,8)"VX$67EEr   c                <    |j                         | _        || _        y r   )rn   rk   rl   )r9   rq   rowss      r   write_recordszMockGSpreadClient.write_records   s    
 #&((*"&r   N)rq   sheets_id.SheetsIdentifier)rq   r|   rw   intr?   r^   )rq   r|   rz   zSequence[Sequence[Any]]r?   rS   )r@   rA   rB   r;   rr   rx   r{   r   r   r   rg   rg   }   sT    G
F'F F 
@	F''' &' 
	'r   rg   rW   quack1quack2QUACK QUACK1NOT QUACK QUACK2z!llmfn_outputs.LLMFnOutputs | None_output_varc                 ~    d a d ad ad at	               at        j                  t        j                  d      d      a
y )Nfake_sheets)rn   *   )rq   rw   )_compiled_function_compiled_lhs_function_compiled_rhs_functionr   r`   _output_sink_varr   SheetsInputsr   SheetsIdentifier_SHEETS_INPUT_VARr   r   r   _reset_globalsr      sH     !!K&( %11&&M:r   c                  J     e Zd Z fdZ	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 ddZ xZS )EndToEndTestsc                    t         |           t               | _        t	        j
                  | j                         t                y r   )supersetUprg   _mock_clientr   testonly_set_clientr   )r9   	__class__s    r   r   zEndToEndTests.setUp   s1    -/**4+<+<=r   c                (   | j                  |t        j                         | j                  ||j	                  d             | j                  t        |j                               t        |j	                  d      j                                      y )Nlistorient)assertIsInstancepandas	DataFrameassertEqualto_dictr   keys)r9   resultsexpected_resultss      r   $_assert_is_expected_pandas_dataframez2EndToEndTests._assert_is_expected_pandas_dataframe   sr     	gv'7'78)7??&?+IJ!&&()/4467	
r   c                B   | j                  |t        j                         | j                  ||j	                                | j                  t        |j                               t        |j	                         j                                      | j                  t        |d             |j                          |j                          | j                  |j                  t        j                         | j                  ||j                  j                  d             y )N_ipython_display_r   r   )r   r   LLMFnOutputsr   as_dictr   r   
assertTruehasattrrK   r   rH   r   r   r   )r9   varr   fake_envs       r   &_assert_output_var_is_expected_resultsz4EndToEndTests._assert_output_var_is_expected_results   s     	c=#=#=> 	)3;;=9!&&()##%&	
 	%89: 	h33V5E5EF)8+@+@+H+HPV+H+WXr   )r   zpandas.DataFramer   Mapping[str, Any]r?   rS   )r   r   r   r   r   rE   r?   rS   )r@   rA   rB   r   r   r   __classcell__)r   s   @r   r   r      sU    
'
;L
	
YY ,Y !	Y
 
Yr   r   __main__c                      e Zd Zd Zd Zd Zy)HelpEndToEndTestsc                    t               }d}t        j                  |      }|j                  |d      }| j	                  t        |      d       y )Nz--helpregistryignoredz"A system for interacting with LLMsr5   r   MagicsEngineexecute_cellassertRegexstrr9   mock_registry
magic_lineenginer   s        r   	test_helpzHelpEndToEndTests.test_help   sH    )+
++]C %%j)<W'KLr   c                    t               }d}t        j                  |      }|j                  |d      }| j	                  t        |      d       y )Nz
run --helpr   r   zusage: palm runr   r   s        r   test_run_helpzHelpEndToEndTests.test_run_help   sH    )+!
++]C %%j)<W'89r   c                    t               }d}t        j                  |      }| j                  t        d      5  |j                  |d       d d d        y # 1 sw Y   y xY w)Nzrun --this_is_an_invalid_flagr   z1unrecognized arguments: --this_is_an_invalid_flagr   )r5   r   r   assertRaisesRegex
SystemExitr   )r9   r   r   r   s       r   
test_errorzHelpEndToEndTests.test_error   sZ    )+4
++]C##K
 	7 
I6	7 	7 	7s   AAN)r@   rA   rB   r   r   r   r   r   r   r   r      s    M:7r   r   c                  T    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zy)RunCmdEndToEndTestsc                   t               }d}t        j                  |      }|j                  |d      }dgdgdgdgdgd}| j	                  ||       |j                  |d      }dgdgdgdgdgd}| j	                  ||       |j                  |d	      }dgdgdgdgdgd}| j	                  ||       | j                  t        j                  j                  |j                         y
)z)Smoke test for executing the run command.zrun --model_type=echor   zquack quackr   
Prompt Num	Input Num
Result NumPrompttext_resultr   r   zline 1
line 2
zline 1
line 2zline 1
line 2

N)
r5   r   r   r   r   r   r   	ModelName
ECHO_MODELr8   )r9   r   r   r   r   r   s         r   test_run_cmdz RunCmdEndToEndTests.test_run_cmd  s7   )+,
++]C%%j-@##$o)?
 	11.> 	2 	
 %%j2DE##'(,-
 	11.> 	2 	

 %%j2FG##)*./
 	11.> 	2 	

 	11<<m>Z>Z[r   c                J   t        j                  t        j                        }t	        |      }t        j                  |      }|j                  dd      }t        j                  ddd      }|j                  j                  j                  d   }| j                  ||       y )	Nr   zSrun --model_type=echo --model=the_best_model --temperature=0.25 --candidate_count=3quackthe_best_modelg      ?   )r   temperaturecandidate_count
model_args)r   create_autospecr   r7   r5   r   r   r   ModelArguments
call_model	call_argskwargsr   )r9   
mock_modelregr   _expected_model_argsactual_model_argss          r   test_model_args_passedz*RunCmdEndToEndTests.test_model_args_passed9  s    ))%//:

+++S9' 
 $22"a
 '11;;BB<P*,?@r   c                    t               }t        j                  |      }|j                  dd      }g dg dg dg dg dd}| j	                  ||       y )	Nr   z#run --model_type=echo --candidate=3r   )r   r   r   )r         )r   r   r   r   r   r5   r   r   r   r   r9   r   r   r   r   s        r   test_candidate_countz(RunCmdEndToEndTests.test_candidate_countL  sd    )+++]C%%1

 $"#16
 	11.> 	2 	
r   c                    t               }t        j                  |      }|j                  dd      }dgdgdgdgdgd}| j	                  ||       y )Nr   z,run --model_type=echo --candidate=3 --uniquer   r   r   r   r   r   s        r   test_uniquezRunCmdEndToEndTests.test_unique^  sn    )+++]C%%:

 ##i#9
 	11.> 	2 	
r   c                   d}t        j                  t                     }|j                  |d      }| j	                  |t
        j                         g dg dg dg dg dd}| j                  ||       | j                  d	| j                  j                         | j                  d
| j                  j                         y )NzNrun --model_type=echo --inputs _INPUT_VAR_ONE _INPUT_VAR_TWO _SHEETS_INPUT_VARr   quack {word})r   r   r   r   r   r   )r   r   r   r         )zquack quack1zquack quack2zquack quack3zquack quack4quack quack5quack quack6r   r   r   r   r   r   r5   r   r   r   r   r   r   r   ri   rj   r9   r   r   r   r   s        r   test_inputs_passedz&RunCmdEndToEndTests.test_inputs_passedp  s    _ 	 ++5F5HI%%j.Agv'7'78,+,
* 	11.> 	2 	
 	(9(9(N(NOT..KKLr   c                   d}t        j                  t                     }|j                  |d      }| j	                  |t
        j                         ddgddgddgddgddgd}| j                  ||	       | j                  d
| j                  j                         | j                  d| j                  j                         y)z Test using --sheets_input_names.9run --model_type=echo --sheets_input_names my_fake_sheetsr   r   r   r   r   r   r   r   my_fake_sheetsNr   r   s        r   test_sheets_input_names_passedz2RunCmdEndToEndTests.test_sheets_input_names_passed  s     Q
++5F5HI%%j.Agv'7'78a&Qa&
 
 	11.> 	2 	
 	)4+<+<+Q+QRD--JJKr   c                   t        j                  t                     }| j                  t        d      5  |j                  dd       d d d        | j                  t        d      5  |j                  dd       d d d        | j                  t        d      5  |j                  dd       d d d        y # 1 sw Y   lxY w# 1 sw Y   GxY w# 1 sw Y   y xY w)	Nr   zrargument --inputs/-i: Error with value "_INPUT_VAR_ONE", got ValueError: Placeholder "not_word" not found in inputz-run --model_type=echo --inputs _INPUT_VAR_ONEquack {not_word}zrargument --inputs/-i: Error with value "_INPUT_VAR_TWO", got ValueError: Placeholder "not_word" not found in inputz-run --model_type=echo --inputs _INPUT_VAR_TWOzuargument --inputs/-i: Error with value "_SHEETS_INPUT_VAR", got ValueError: Placeholder "not_word" not found in inputz0run --model_type=echo --inputs _SHEETS_INPUT_VARr   r   r5   r   r   r   r9   r   s     r   )test_validate_inputs_against_placeholdersz=RunCmdEndToEndTests.test_validate_inputs_against_placeholders  s    ++5F5HI##I
 
	 ?"
	 ##I
 
	 ?"
	 ##I
 
	 B"
	 
	1
	 
	
	 
	
	 
	s#   B3&B?C3B<?CCc                    t        j                  t                     }| j                  t        d      5  |j                  dd       d d d        y # 1 sw Y   y xY w)Nr   zargument --sheets_input_names/-si: Error with value "my_fake_sheets", got ValueError: Placeholder "not_word" not found in inputr   r   r   r   s     r   0test_validate_sheets_inputs_against_placeholderszDRunCmdEndToEndTests.test_validate_sheets_inputs_against_placeholders  sW    ++5F5HI##"
 	 K"	 	 	s   AAc                    d}t        j                  t                     }|j                  |d      }| j	                  |t
        j                         dgdgdgdgdgdgdgd}| j                  ||	       y )
NzBrun --model_type=echo | add_length | repeat | add_length_decoratedr   r   r   r   
   
quackquack)r   r   r   r   r"   r$   r   r   )r   r   r5   r   r   r   r   r   r   s        r   test_post_processz%RunCmdEndToEndTests.test_post_process  s    Y
++5F5HI%%j':gv'7'78##i#%'D(>
 	11.> 	2 	
r   c                :   d}t               }t        j                  t               |      }|j	                  |d      }| j                  t        t        j                         dgdgdgdgdgdgd}| j                  ||       | j                  t        ||	       y )
NzArun --model_type=echo --outputs _output_var | add_length | repeatr   envr   r   r   r  r   r   r   r   r"   r   r   r   r   r   )rE   r   r   r5   r   r   r   r   r   r   r   )r9   r   r   r   r   r   s         r   test_outputsz RunCmdEndToEndTests.test_outputs  s     Y
!#++5F5HhW%%j':k=+E+EF##i#(>
 	11.> 	2 	
 	33- 	4 	
r   c                   d}t        j                  t                     }|j                  |d      }| j	                  t
        j                         | j                  t
        j                  t        j                         dgdgdgdgdgdgd}| j                  ||       | j                  |t
        j                  j                         j                  d	
             y )NzFrun --model_type=echo --outputs _output_sink_var | add_length | repeatr   r   r   r   r  r  r   r   r   )r   r   r5   r   assertIsNotNoner   rc   r   r   r   r   r   as_pandas_dataframer   r   s        r   test_outputs_sinkz%RunCmdEndToEndTests.test_outputs_sink  s     ^
++5F5HI%%j':-556.668R8RS##i#(>
 	11.> 	2 	
 	$$88:BB&BQ	
r   c                   d}t        j                  t                     }|j                  |d      }| j	                  d| j
                  j                         g dg dg}| j	                  || j
                  j                         y )NzXrun --model_type=echo --sheets_output_names my_fake_output_sheets  | add_length | repeatr   r   my_fake_output_sheetsr  )r   r   r   r   r   r  )r   r   r5   r   r   r   rk   rl   )r9   r   r   r   expected_rowss        r   test_sheets_outputs_namesz-RunCmdEndToEndTests.test_sheets_outputs_names*  s    # 	 ++5F5HI
G40$2C2C2V2VW 0

 	(9(9(L(LMr   N)r@   rA   rB   r   r   r   r   r   r   r   r   r  r	  r  r  r   r   r   r   r   	  sF    ,\\A&
$
$!MFL8$L
$
6
2Nr   r   c                      e Zd Zd Zy)CompileCmdEndToEndTestsc                    t               }t        j                  t               |      }|j	                  dd      }| j                  t        t        j                         t        ddgi      }dgdgdgdgdgd}| j                  |||	       y )
Nr  z,compile _compiled_function --model_type=echor   rW   z
LOUD QUACKr   zquack LOUD QUACKr   r  )
rE   r   r   r5   r   r   r   r   LLMFunctionr   )r9   r   r   r   rc   expected_outputss         r   test_compile_cmdz(CompileCmdEndToEndTests.test_compile_cmdF  s    !#++5F5HhW NP^_ 	0,2J2JK$f|n%=>##)*./
 	33*:X 	4 	
r   N)r@   rA   rB   r  r   r   r   r  r  D  s    
r   r  c                      e Zd Zd Zd Zy)CompareCmdEndToEndTestsc                P   t               }t        j                  t               |      }|j	                  dd      }|j	                  dd      }|j	                  dd      }ddgdd	gddgd
did
digddgddgddgd}| j                  ||       | j                  t        ||       y )Nr  0compile _compiled_lhs_function --model_type=echoleft quack {word}0compile _compiled_rhs_function --model_type=echoright quack {word}zccompare _compiled_lhs_function _compiled_rhs_function --inputs _INPUT_VAR_ONE --outputs _output_varr   r   r   rW   r~   r   left quack quack1left quack quack2right quack quack1right quack quack2F)r   r   r   Prompt vars"_compiled_lhs_function_text_result"_compiled_rhs_function_text_resultis_equalr   r  rE   r   r   r5   r   r   r   r   r9   r   r   r   r   r   s         r   (test_compare_cmd_with_default_compare_fnz@CompareCmdEndToEndTests.test_compare_cmd_with_default_compare_fn_  s    !#++5F5HhW >
 > 
 %%8 
 a&Qa&#X.0BC##3
 %$3 
 	11.> 	2 	
 	33- 	4 	
r   c           	     V   t               }t        j                  t               |      }|j	                  dd      }|j	                  dd      }|j	                  dd      }ddgdd	gddgd
did
digddgddgddgddgd}| j                  ||       | j                  t        ||       y )Nr  r  r  r  r  zcompare _compiled_lhs_function _compiled_rhs_function --inputs _INPUT_VAR_ONE --outputs _output_var --compare_fn concat get_sum_of_lengthsr   r   r   rW   r~   r   r  r   r!  r"  z$left quack quack1 right quack quack1z$left quack quack2 right quack quack2#   )r   r   r   r#  r$  r%  r1   r.   r   r  r'  r(  s         r   'test_compare_cmd_with_custom_compare_fnz?CompareCmdEndToEndTests.test_compare_cmd_with_custom_compare_fn  s   !#++5F5HhW >
 > 
 %%& 
 a&Qa&#X.0BC##3
 %$3
 76 $&r(#
& 	11.> 	2 	
 	33- 	4 	
r   N)r@   rA   rB   r)  r,  r   r   r   r  r  ]  s    /
b4
r   r  c                      e Zd Zd Zy)EvalCmdEndToEndTestsc           	        t               }t        j                  t               |      }|j	                  dd      }ddgddgddgddiddigd	d
gd	dgddgddgd}| j                  ||       | j                  t        ||       y )Nr  zeval --model_type=echo --ground_truth _GROUND_TRUTH_VAR --inputs _INPUT_VAR_ONE --outputs _output_var --compare_fn get_sum_of_lengths my_is_equal_fn | to_upperr   r   r   rW   r~   r   r   zQUACK QUACK2r         TF)r   r   r   r#  actual_text_resultground_truth_text_resultr.   r3   r   r  r'  )r9   r   r   r   r   s        r   test_eval_cmdz"EvalCmdEndToEndTests.test_eval_cmd  s    !#++5F5HhW %%@ 
 a&Qa&#X.0BC#1>"B)79K(L#%r(#Um	
 	11.> 	2 	
 	33- 	4 	
r   N)r@   rA   rB   r4  r   r   r   r.  r.    s    &
r   r.  )r   r   r?   r   )r!   r   r?   r}   )r!   r   r?   r   )r,   r   r-   r   r?   r}   )r,   r   r-   r   r?   r   )r,   r   r-   r   r?   boolrR   )G
__future__r   systypingr   r   r   r   unittestr   abslr	   absl.testingr
   google.generativeai.notebookr   r   r   r   r   r   r    google.generativeai.notebook.libr   r   r   r   r   r   r   r   r   r"   post_process_add_fnr$   post_process_replace_fnr&   r)   r.   r1   r3   ModelRegistryr5   
IPythonEnvrE   LLMFnInputsSourcerU   LLMFnOutputsSinkr`   GSpreadClientrg   _INPUT_VAR_ONE_INPUT_VAR_TWOr   _GROUND_TRUTH_VARr   __annotations__r   r   TestCaser   patchdictmodulesr@   r   r   r  r  r.  mainr   r   r   <module>rN     s   # 
 3 3   ! 7 4 6 7 ; 2 5 9 @ : 2  ( + +  '' ( ++ , ++ ,
	44 	D[++ D"F*<< Fm44 '44 '@ 8X./!#  $%78  26. 5"$ ,'YH%% 'YT z3;;x+@AB7 7 C7> z3;;x+@ABvN- vN CvNt	 z3;;x+@AB
m 
 C
0 z3;;x+@ABf
m f
 Cf
T z3;;x+@AB'
= '
 C'
T zHMMO r   