
    +#hA                     (   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlZe j                  j                  d      r6e j                  j                  e j                  j                  d      d      ZndZddiZd Zd	 Zd
 Zd Zd Zd Zd ZddZy)    N)HasherDSP_NOTEBOOK_CACHEDIRcompilerzcache/compilerzft-zvEdzQVQ5xwlxvNPrxl6kpnwz*ada:ft-stanfordpraglab-2023-02-09-19-50-49c                    t         j                  j                  r| t        v r	t        |    S d|  }t	        |       t        j                  |j                         t
        j                  d      }|j                  j                  d      j                         }	 t        j                  |      }|d   dk(  r|d   S |d   dv r/t	        d	|  d
       t        j                  d       t!        |       S 	 y#  Y yxY w)Nzopenai api fine_tunes.get -i F)stdoutcheckzutf-8status	succeededfine_tuned_model)pendingrunningz2Compiling, run ```openai api fine_tunes.follow -i z``` for details...<   )dspsettingsforce_reuse_cached_compilationcompilations_assumed_to_existprint
subprocessrunsplitPIPEr   decodestripujsonloadstimesleepopenai_check_finetune)jobnamecommandresultoutputs       T/var/www/html/sandstorm/venv/lib/python3.12/site-packages/dsp/primitives/compiler.pyr   r      s    
||22wB_7_,W55/y;G	'N^^GMMOJOO5QF]]!!'*002F
V$({*,--(55FwiOabcJJrN(11 6 s   !C4 <5C4 4C8c           
         t        |      t        |      z   t        |j                        k(  sJ t        j                  d
i |D ci c]  }|| |   
 c}dg i} ||d      }| |d      }|j                  t        |      d  }|dd  D ]%  }	|d|	j                   d| |	j
                     z   z  }' d|z   dz   }||d	S c c}w )NdemosF)show_guidelinesr      z

  </s>)prompt
completion )lenfieldsr   Examplenameoutput_variable)
yinputsoutputstemplatefy_r*   r+   output_fieldsfields
             r#   convert_to_training_point2r:   .   s    v;W%X__)====		;0A1Q40	;	;Bb%0F71:JOOCKL1Mqr" FUZZL*Qu/D/D-EEE
F z!G+JJ77 1s   B?c           
      H   g }t        j                   |      D ]a  } | |      }||j                  D ]E  }|d   |d   |d   |d   f\  }}}}	|j                  t        |j	                  |      ||	|             G c t        j                  d      }
|
j                  |       |S )Nr0   r5   r3   r4   r   )tqdmcompiling_stagesappendr:   getrandomRandomshuffle)programinput_examplestraining_datainput_example
predictionstager0   r5   r3   r4   rs              r#   simulaterJ   >   s    M>2 r]+
!#44 r27-zARTYZbTcejkteu2u/h$$%?
t@TV\^ego%pqrr 	aAIIm    c                 8   t        |       }d| d| d}t        |       t        j                  |j	                         t        j
                  t        j
                        }|j                  j                         j                         j                         x}r_d|j                         v r|j	                         d   }n9|j                  j                         j                         j                         x}r_t               t        |      }|sJ |       ||fS )Nz openai api fine_tunes.create -t z -m zG --n_epochs 4 --learning_rate_multiplier 0.05 --no_check_if_files_exist)r   stderrzcreated fine-tune:)name_to_pathr   r   Popenr   r   r   readliner   r   lowerr   )r0   targettraining_data_pathr    processliner   fts           r#   openai_finetune_rX   P   s    %d+ 33E2Fd6(  SZ  ]G	'N w}}zz_G..))+224::<
<$
<4::</jjl2&G ..))+224::<
<$
<  
'N 
w	'BMrM2 R=rK   c                    t        |        t        |       }|dz  }	 t        |      5 }t        j                  |j                               \  }}d d d        t              r|fS 	 t        | |      \  }}t        |d      5 }|j                  t        j                  ||f      dz          d d d        ||fS # 1 sw Y   jxY w#  Y axY w# 1 sw Y   ||fS xY w)Nz.modelw
)
r   rO   openr   r   rQ   r   rX   writedumps)r0   rS   rT   r6   r   rW   s         r#   openai_finetuner_   t   s    	$K%d+("$% 	4++ajjl3KGR	4 !)B; *
 #40KGR	 #	& 3!	WbM*T123 B;	4 	4
3 B;s.   C 'B8C *C8C=C CCc                     t         j                  j                  t              st        j                  t               t         j                  j                  t        |  d      }|S )Nz.jsonl)ospathexiststraining_data_directorymakedirsjoin)r0   rT   s     r#   rO   rO      sB    77>>12
+,&=$vOrK   c                 H   t        j                  |       }t        |      }t        |d      5 }| D ])  }|j	                  t        j                  |      dz          + 	 d d d        t        ||      \  }}t        |       t        j                  |d      }|S # 1 sw Y   <xY w)NrZ   r[   r)   )modelstop)r   hashrO   r\   r]   r   r^   r_   r   r   GPT3)rE   rS   r0   rT   r6   rV   r   rW   s           r#   finetunerl      s    ;;}%D%d+	 #	& .!! 	.DGGEKK%,-	.. "$/KGR	"I		)BI. .s   /BB!c                 V     t         |      }t        ||       fd}|_        |S )N)rS   c                      t         j                  j                  d      5   | i |cd d d        S # 1 sw Y   y xY w)NF)compiled_lm	compiling)r   r   context)argskwargsro   rC   s     r#   compiled_programz!compile.<locals>.compiled_program   s<    \\!!kU!K 	,D+F+	, 	, 	,s   5>)rJ   rl   lm)rC   examplesrS   rE   rt   ro   s   `    @r#   compilerw      s1    Wh/M=8K, &rK   )ada)ra   r@   r   r   r<   r   datasets.fingerprintr   r   environr?   rb   rf   rd   r   r   r:   rJ   rX   r_   rO   rl   rw   r,   rK   r#   <module>r{      s    	      ' 
::>>)* ggll2::>>:Q+RT^_.  =>jk 28 $!H.	rK   