From 0164158b93f6a3496e82656704dad0c1dee3e426 Mon Sep 17 00:00:00 2001 From: David Pugh Date: Sun, 20 Jan 2019 17:47:00 +0000 Subject: [PATCH] Adding fix for expected Cython errors - closes #58 --- .travis.yml | 12 +++---- Pipfile | 1 - appveyor.yml | 10 +++--- .../algorithms/markov_chain_monte_carlo.py | 5 +-- src/MTfit/probability/probability.py | 36 ++++++++++--------- src/MTfit/sampling.py | 4 +-- 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7d425f0..14c1791 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,11 +59,11 @@ jobs: script: tox -e test-py36 python: 3.6 name: Unit Tests - - stage: test - script: python ci/travis_appveyor_build.py - python: 3.6 - name: Unit Tests (Windows) - if: branch in (master, develop) OR tag IS present + # - stage: test + # script: python ci/travis_appveyor_build.py + # python: 3.6 + # name: Unit Tests (Windows) + # if: branch in (master, develop) OR tag IS present - stage: examples-test script: tox -e examples-py27 python: 2.7 @@ -79,7 +79,7 @@ jobs: - stage: build python: 3.6 name: Build - before_deploy: python ci/get_appveyor_wheels.py + # before_deploy: python ci/get_appveyor_wheels.py deploy: - provider: pypi user: "$PYPI_USERNAME" diff --git a/Pipfile b/Pipfile index 0f5cd30..df850c7 100644 --- a/Pipfile +++ b/Pipfile @@ -9,4 +9,3 @@ scipy = "==1.0.0" matplotlib = "==2.1.0" [dev-packages] - diff --git a/appveyor.yml b/appveyor.yml index 6f40136..ece1b19 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -25,7 +25,7 @@ environment: install: # We need wheel installed to build wheels - - cmd: "%PYTHON%\\python.exe -m pip install wheel tox numpy cython" + - cmd: "%PYTHON%\\python.exe -m pip install wheel tox numpy cython twine" build: off @@ -50,12 +50,14 @@ after_test: # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct # interpreter - ps: if ($(& $env:PYTHON_EXE .\ci\get_version.py) -ne '') {& $env:PYTHON_EXE setup.py bdist_wheel} ELSE {echo "Not building because this is not a tag build"} - - ps: if ($LAST_EXIT_CODE -eq 'True') {ls dist} + - ps: if (& $env:PYTHON_EXE .\ci\get_version.py) {ls dist} + artifacts: # bdist_wheel puts your built wheel in the dist directory - path: dist\* -#on_success: +on_success: # You can use this step to upload your artifacts to a public website. # See Appveyor's documentation for more details. Or you can simply -# access your wheels from the Appveyor "artifacts" tab for your build. \ No newline at end of file +# access your wheels from the Appveyor "artifacts" tab for your build. + - ps if ($(& $env:PYTHON_EXE .\ci\get_version.py) -ne '') {& $env:PYTHON_EXE setup.py bdist_wheel} ELSE {echo "Not building because this is not a tag build"} \ No newline at end of file diff --git a/src/MTfit/algorithms/markov_chain_monte_carlo.py b/src/MTfit/algorithms/markov_chain_monte_carlo.py index 64be869..cc69773 100644 --- a/src/MTfit/algorithms/markov_chain_monte_carlo.py +++ b/src/MTfit/algorithms/markov_chain_monte_carlo.py @@ -1516,7 +1516,7 @@ def new_sample(self, jump=0.0, gaussian_jump=False): gaussian_jump=gaussian_jump) return mt except Exception: - logging.exception('Cython error') + logging.exception('WARNING: Error running cython code, resorting to python code') else: logger.info(C_EXTENSION_FALLBACK_LOG_MSG) # Otherwise/Fallback to use python code @@ -1611,7 +1611,8 @@ def _acceptance_check(self, xi_1, ln_pi_1, scale_factori_1=False): return xi_1, ln_pi_1, scale_factori_1[index], index return xi_1, ln_pi_1, False, index except Exception: - logger.exception('Cython Error') + if not isinstance(xi_1, dict): + logger.exception('WARNING: Error running cython code, resorting to python code') else: logger.info(C_EXTENSION_FALLBACK_LOG_MSG) # Otherwise use/fallback to Python code diff --git a/src/MTfit/probability/probability.py b/src/MTfit/probability/probability.py index 065512e..4e830b3 100644 --- a/src/MTfit/probability/probability.py +++ b/src/MTfit/probability/probability.py @@ -137,7 +137,7 @@ def polarity_ln_pdf(a, mt, sigma, incorrect_polarity_probability=0.0, _use_c=Non except Exception as e: # Run using python # Testing C code - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') if _C_LIB_TESTS: raise e else: @@ -171,7 +171,7 @@ def polarity_ln_pdf(a, mt, sigma, incorrect_polarity_probability=0.0, _use_c=Non ln_p = cprobability.ln_prod(ln_p) except Exception: if cprobability: - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') ln_p = np.sum(ln_p, 0) if isinstance(ln_p, np.ndarray): ln_p[np.isnan(ln_p)] = -np.inf @@ -255,7 +255,7 @@ def polarity_probability_ln_pdf(a, mt, positive_probability, negative_probabilit except Exception as e: # Run using python # Testing C code - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') if _C_LIB_TESTS: raise e else: @@ -360,7 +360,7 @@ def amplitude_ratio_ln_pdf(ratio, mt, a_x, a_y, percentage_error_x, percentage_e except Exception as e: # Run using python # Testing C code - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') if _C_LIB_TESTS: raise e else: @@ -394,7 +394,7 @@ def amplitude_ratio_ln_pdf(ratio, mt, a_x, a_y, percentage_error_x, percentage_e ln_p = cprobability.ln_prod(ln_p) except Exception: if cprobability: - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') ln_p = np.sum(ln_p, 0) if isinstance(ln_p, np.ndarray): ln_p[np.isnan(ln_p)] = -np.inf @@ -479,7 +479,7 @@ def relative_amplitude_ratio_ln_pdf(x_1, x_2, mt_1, mt_2, a_1, a_2, percentage_e except Exception as e: # Run using python # Testing C code - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') if _C_LIB_TESTS: raise e else: @@ -826,7 +826,7 @@ def dkl(ln_probability_p, ln_probability_q, dV=1.0): try: return cprobability.dkl(ln_probability_p.copy(), ln_probability_q.copy(), dV) except Exception: - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') else: logger.info(C_EXTENSION_FALLBACK_LOG_MSG) ind = ln_probability_p > -np.inf @@ -845,7 +845,7 @@ def dkl(ln_probability_p, ln_probability_q, dV=1.0): ln_probability_q[ind]*probability_p[ind]) * dV -def ln_marginalise(ln_pdf, axis=0, dV=1.0): +def ln_marginalise(ln_pdf, axis=0, dV=1.0, _cprob_err=True): """ Marginalise the pdf from the log pdf input @@ -870,7 +870,8 @@ def ln_marginalise(ln_pdf, axis=0, dV=1.0): return cprobability.ln_marginalise(ln_pdf._ln_pdf.astype(np.float64)) return cprobability.ln_marginalise(ln_pdf.astype(np.float64)) except Exception: - logger.exception('Error running cython code') + if _cprob_err: + logger.exception('WARNING: Error running cython code, resorting to python code') else: logger.info(C_EXTENSION_FALLBACK_LOG_MSG) # scale and then marginalise: @@ -922,7 +923,7 @@ def ln_normalise(ln_pdf, dV=1): normalised_ln_pdf = cprobability.ln_normalise(ln_pdf) return normalised_ln_pdf except Exception: - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') else: logger.info(C_EXTENSION_FALLBACK_LOG_MSG) # scale and then marginalise: @@ -993,7 +994,7 @@ def dkl_estimate(ln_pdf, V, N): try: return cprobability.dkl_uniform(ln_pdf.copy(), V, dV) except Exception: - logger.exception('Error running cython code') + logger.exception('WARNING: Error running cython code, resorting to python code') else: logger.info(C_EXTENSION_FALLBACK_LOG_MSG) ind = ln_pdf > -np.inf @@ -1263,17 +1264,18 @@ def output(self, normalise=True): return self.marginalise().normalise() return self.marginalise() - def exp(self): + def exp(self, _cprob_err=True): if cprobability: try: return cprobability.ln_exp(self._ln_pdf) except Exception: - logger.exception('Error running cython code') + if _cprob_err: + logger.exception('WARNING: Error running cython code, resorting to python code') else: logger.info(C_EXTENSION_FALLBACK_LOG_MSG) return np.exp(self._ln_pdf) - def nonzero(self, discard=100000., n_samples=0): + def nonzero(self, discard=100000., n_samples=0, _cprob_err=True): """ Return the non-zero indices of the pdf @@ -1286,7 +1288,7 @@ def nonzero(self, discard=100000., n_samples=0): discard: float - discard scale [default = 100000.] n_samples: integer - number of samples generated [default = 0] """ - ln_pdf = np.array(self.marginalise(axis=0)._ln_pdf).flatten() + ln_pdf = np.array(self.marginalise(axis=0, _cprob_err=_cprob_err)._ln_pdf).flatten() m_val = -np.inf if n_samples > 0 and discard > 0: m_val = max(ln_pdf) - np.log(discard*n_samples) @@ -1308,7 +1310,7 @@ def normalise(self, dV=False): new._ln_pdf = ln_normalise(self._ln_pdf, self.dV) return new - def marginalise(self, axis=0, dV=False): + def marginalise(self, axis=0, dV=False, _cprob_err=True): """ Marginalise the pdf object over a given axis @@ -1322,7 +1324,7 @@ def marginalise(self, axis=0, dV=False): if dV: self._set_dv(dV) new = self.__class__(dV=self.dV) - new._ln_pdf = ln_marginalise(self._ln_pdf, axis=axis, dV=self.dV) + new._ln_pdf = ln_marginalise(self._ln_pdf, axis=axis, dV=self.dV, _cprob_err=_cprob_err) return new def append(self, other, axis=1): diff --git a/src/MTfit/sampling.py b/src/MTfit/sampling.py index 95c1ddf..61536d6 100644 --- a/src/MTfit/sampling.py +++ b/src/MTfit/sampling.py @@ -170,7 +170,7 @@ def output(self, normalise=True, convert=False, n_samples=0, discard=10000, mcmc # Check if there are samples if len(ln_pdf): # Get non_zero samples - non_zero = ln_pdf.nonzero(discard=discard, n_samples=n_samples) + non_zero = ln_pdf.nonzero(discard=discard, n_samples=n_samples, _cprob_err=False) if discard and n_samples: output_string += 'After discard, '+str(non_zero.shape[0])+' samples remain\n\n' if len(ln_pdf.shape) > 1: @@ -428,7 +428,7 @@ def ln_bayesian_evidence(output, n_samples, prior=_6sphere_prior): p = prior(output['g'], output['d']) if not isinstance(output['ln_pdf'], LnPDF): output['ln_pdf'] = LnPDF(output['ln_pdf']) - return np.log((output['ln_pdf']+np.log(p)-output['ln_pdf']._ln_pdf.max()).exp().sum())+output['ln_pdf']._ln_pdf.max()-np.log(n_samples) + return np.log((output['ln_pdf']+np.log(p)-output['ln_pdf']._ln_pdf.max()).exp(_cprob_err=False).sum())+output['ln_pdf']._ln_pdf.max()-np.log(n_samples) def _convert(moment_tensors, i=None):