Блог

У меня есть фрейм данных:

 import pandas as pd df = pd.DataFrame({  'PC1' : [0.035182, 0.001649, -0.080456, 0.056460, 0.017737, -0.005615, 0.033691, 0.547145, -0.022938, -0.059511],   'PC2': [0.034898, 0.001629, -0.083374, 0.053976, 0.017603,-0.005902, 0.006798, 0.250167, -0.137955, -0.313852],   'PC3': [0.032212, 0.001591, -0.067145, 0.047500, 0.015782, -0.003079, 0.012376, 0.302485, -0.063795, -0.124957],   'PC4' : [-0.000632,0.001268,0.063346,-0.026841,-0.009790,0.029897,-0.018870,-0.449655,0.081417,-0.327028],   'PC5' : [0.020340,0.001734,-0.050830,0.008507,0.007470,0.013534,0.100008,1.083280,0.298315,0.736401],   'PC6' : [0.027012,0.001507,-0.036496,0.032256,0.012207,0.005451,0.081582,0.959821,0.337683,0.758737],   'PC7' : [0.027903,0.001625,-0.041970,0.039854,0.014676,0.002364,0.045583,0.620938,0.116647,0.214294],   'PC8' : [0.013828,-0.015836,-0.117484,-0.208933,-0.162090,-0.190467,-0.075784,-0.481607,-0.213148,-0.401169],   'PC9' : [0.009378,0.002712,-0.148531,0.040901,0.011923,-0.000078,-0.055367,-0.661758,0.242363,-0.392438],   'PC10' : [-0.002740,-0.000234,0.060118,0.027855,0.016309,0.009850,-0.108481,-1.560047,0.198750,-0.793165],   'PC11' : [-2.876278,-0.437754,0.764775,-0.627843,0.391284,0.090675,-0.007820,0.342359,0.052004,-0.200808],   'PC12' : [-2.411929,-0.414697,0.415683,-0.426348,0.302643,-0.160550,-0.051552,1.086344,-0.275267,1.219304] })  df.head()  

Я применил функцию «pd.cut» к каждому столбцу в фрейме данных. qcut в основном является функцией дискретизации на основе квантилей.

 cuts = []  for col in df.columns:  cuts.append(pd.qcut(df[col], 2, labels=None, retbins=False, precision=3, duplicates='raise'))  X = pd.concat(cuts, axis=1)  

Затем я хочу взять только 2 значения, которые уникальны для каждого столбца PC1, PC2,..... PCn.

 uniq = [] for i in x.columns:  uniq.append(x[i].unique())  unique = pd.DataFrame(uniq) unique  

Результат выглядит так:

Уникальная переменная состоит из 2 значений в виде (a,b]

Затем я хочу настроить класс transformer для создания новых категориальных фиктивных функций.

 # custom transformer class to create new categorical dummy features class WoE_Binning(BaseEstimator, TransformerMixin):  def __init__(self, X): # no *args or *kargs  self.X = X  def fit(self, X, y = None):  return self #nothing else to do  def transform(self, X):    X_new['PC1:0.00969 - 0.547'] = np.where((X['PC1'] gt; 0.00969) amp; (X['PC1'] lt;= 0.547), 1, 0)  X_new['PC1:-0.0815 - 0.00969'] = np.where((X['PC1'] gt; 0.0815 ) amp; (X['PC1'] lt;= 0.00969), 1, 0)  X_new['PC2:0.00421 - 0.25'] = np.where((X['PC2'] gt; 0.00421) amp; (X['PC2'] lt;= 0.25), 1, 0)  X_new['PC2:-0.315 - 0.00421'] = np.where((X['PC2'] gt; 0.315) amp; (X['PC2'] lt;= 0.00421), 1, 0)  X_new['PC3:0.00698 - 0.302'] = np.where((X['PC3'] gt; 7.071) amp; (X['PC3'] lt;= 10.374), 1, 0)  X_new['PC3:-0.126 - 0.00698'] = np.where((X['PC3'] gt; 10.374) amp; (X['PC3'] lt;= 13.676), 1, 0)  X_new['PC4:-0.00521 - 0.0814'] = np.where((X['PC4'] gt; 7.071) amp; (X['PC4'] lt;= 10.374), 1, 0)  X_new['PC4:-0.451 - -0.00521'] = np.where((X['PC4'] gt; 10.374) amp; (X['PC4'] lt;= 13.676), 1, 0)   X_new['PC5:0.0169 - 1.083'] = np.where((X['PC5'] gt; 7.071) amp; (X['PC5'] lt;= 10.374), 1, 0)  X_new['PC5:-0.0518 - 0.0169'] = np.where((X['PC5'] gt; 10.374) amp; (X['PC5'] lt;= 13.676), 1, 0)   X_new['PC6:-0.0375 - 0.0296'] = np.where((X['PC6'] gt; 7.071) amp; (X['PC6'] lt;= 10.374), 1, 0)  X_new['PC6:0.0296 - 0.96'] = np.where((X['PC6'] gt; 10.374) amp; (X['PC6'] lt;= 13.676), 1, 0)   X_new['PC7:0.0296 - 0.96'] = np.where((X['PC7'] gt; 7.071) amp; (X['PC7'] lt;= 10.374), 1, 0)  X_new['PC7:-0.043000000000000003 - 0.0339'] = np.where((X['PC7'] gt; 10.374) amp; (X['PC7'] lt;= 13.676), 1, 0)  X_new['PC8:-0.176 - 0.0138'] = np.where((X['PC8'] gt; 7.071) amp; (X['PC8'] lt;= 10.374), 1, 0)  X_new['PC8:-0.483 - -0.176'] = np.where((X['PC8'] gt; 10.374) amp; (X['PC8'] lt;= 13.676), 1, 0)  X_new['PC9:0.00132 - 0.242'] = np.where((X['PC9'] gt; 7.071) amp; (X['PC9'] lt;= 10.374), 1, 0)  X_new['PC9:-0.663 - 0.00132'] = np.where((X['PC9'] gt; 10.374) amp; (X['PC9'] lt;= 13.676), 1, 0)  X_new['PC10:-1.561 - 0.00481'] = np.where((X['PC10'] gt; 7.071) amp; (X['PC10'] lt;= 10.374), 1, 0)  X_new['PC10:0.00481 - 0.199'] = np.where((X['PC10'] gt; 10.374) amp; (X['PC10'] lt;= 13.676), 1, 0)   X_new['PC11:-2.877 - 0.0221'] = np.where((X['PC11'] gt; 7.071) amp; (X['PC11'] lt;= 10.374), 1, 0)  X_new['PC11:0.0221 - 0.765'] = np.where((X['PC11'] gt; 10.374) amp; (X['PC11'] lt;= 13.676), 1, 0)   X_new['PC12:-2.413 - -0.106'] = np.where((X['PC12'] gt; 7.071) amp; (X['PC12'] lt;= 10.374), 1, 0)  X_new['PC12:-0.106 - 1.219'] = np.where((X['PC12'] gt; 10.374) amp; (X['PC12'] lt;= 13.676), 1, 0)   X_new.drop(columns = ref_categories, inplace = True)  return X_new  

Есть ли более быстрый и простой способ ввести (a,b] в уникальную переменную и имя столбца среза X (PC1, PC2, …PCn) в :

 X_new['PC12:-0.106 - 1.219'] = np.where((X['PC12'] gt; a ) amp; (X['PC12'] lt;= b ), 1, 0)   

Учитывая df фреймы данных, и unique вы могли бы сделать

 X_new = pd.concat(  (  ((interval.left lt; df[col]) amp; (df[col] lt;= interval.right))  .rename(f"{col}: {interval.left} - {interval.right}")  for i, col in enumerate(df.columns) for interval in unique.iloc[:, i]  ),  axis=1 ).astype(int)  

или

 X_new = pd.concat(  (  pd.cut(df[col], [interval.left, interval.right])  .rename(f"{col}: {interval.left} - {interval.right}")  for i, col in enumerate(df.columns) for interval in unique.iloc[:, i]  ),  axis=1 ).notna().astype(int)  

Результат:

 PC1: 0.00969 - 0.547 ... PC12: -0.106 - 1.219 0 1 ... 0 1 0 ... 0 2 0 ... 1 3 1 ... 0 4 1 ... 1 5 0 ... 0 6 1 ... 1 7 0 ... 1 8 0 ... 0 9 0 ... 0  [10 rows x 24 columns]  

Или стройте unique с именами столбцов либо таким образом

 unique = pd.concat(  (pd.DataFrame(X[col].unique(), columns=[col]) for col in X.columns),  axis=1 )  

или, если вам это не нужно X , вот так

 unique = pd.DataFrame(  {  col: pd.qcut(  df[col], 2, labels=None, retbins=False, precision=3, duplicates='raise'  ).unique()  for col in df.columns  } )  

а потом сделайте

 X_new = pd.concat(  (  ((interval.left lt; df[col]) amp; (df[col] lt;= interval.right))  .rename(f"{col}: {interval.left} - {interval.right}")  for col in unique.columns for interval in unique[col]  ),  axis=1 ).astype(int)  

и т.д.